mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
Merge 76bf91316e into 3577aa2d94
This commit is contained in:
commit
bfd02b776b
14 changed files with 1835 additions and 26 deletions
File diff suppressed because one or more lines are too long
365
litellm/proxy/common_utils/codex_catalog_source.py
Normal file
365
litellm/proxy/common_utils/codex_catalog_source.py
Normal file
|
|
@ -0,0 +1,365 @@
|
|||
"""Codex's own model catalog for the Codex release that is asking.
|
||||
|
||||
Codex pointed at ``/v1/models`` as its ``model_catalog_url`` replaces its bundled catalog with ours, so a
|
||||
model missing from the stock entries we serve runs on Codex's fallback metadata (no apply_patch tool, no
|
||||
parallel tool calls, a generic prompt). With an OpenAI key at hand the catalog for the exact
|
||||
``client_version`` asking comes from OpenAI's ``CODEX_CATALOG_OPENAI_URL``, minus the ``base_instructions``
|
||||
copy of each entry's ``model_messages.instructions_template`` (every tagged Codex reads the template first,
|
||||
and the copy doubles a catalog that Codex before 0.166.0 drops past 1 MiB), and is fetched again after
|
||||
``CODEX_CATALOG_REFRESH_SECONDS``, a failed refresh keeping it. Otherwise, or when OpenAI fails before it has
|
||||
answered once, it is the release's entries openai/codex publishes at
|
||||
``https://raw.githubusercontent.com/openai/codex/rust-v<version>/codex-rs/models-manager/models.json``
|
||||
(Apache-2.0) with its ``prompt.md`` beside them, kept for the life of the process since a release tag never
|
||||
changes, and OpenAI is asked again after ``CODEX_CATALOG_RETRY_SECONDS``. Either catalog is laid over the
|
||||
vendored entries, so a model it lacks keeps the entry LiteLLM ships for it. A version without a tag (before
|
||||
0.120, or not released yet), a fetch that fails, a body that does not parse, a request the fetch outlasts by
|
||||
``CODEX_CATALOG_RESPONSE_DEADLINE_SECONDS`` (Codex gives up on its catalog after five),
|
||||
``LITELLM_OFFLINE_MODE`` and ``LITELLM_LOCAL_CODEX_CATALOG=True`` all serve the vendored pair instead. A
|
||||
failed fetch is tried again in the background after ``CODEX_CATALOG_RETRY_SECONDS`` while the last catalog
|
||||
fetched for that release, else the vendored pair, keeps answering, and at most
|
||||
``CODEX_CATALOG_CACHED_VERSIONS`` fetches start per retry window, so made-up versions cannot turn requests
|
||||
into outbound calls. ``LITELLM_CODEX_CATALOG_BASE_URL`` points the fetch at a mirror of the openai/codex raw
|
||||
tree, and an empty ``LITELLM_CODEX_CATALOG_OPENAI_URL`` keeps the OpenAI key home by skipping OpenAI's
|
||||
catalog.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
from collections import OrderedDict
|
||||
from collections.abc import Awaitable, Callable, Mapping
|
||||
from dataclasses import dataclass
|
||||
from functools import cache
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Protocol, TypeAlias
|
||||
|
||||
import httpx
|
||||
from pydantic import ConfigDict, ValidationError
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.llms.custom_httpx.http_handler import HTTPResponseLimitError
|
||||
from litellm.proxy.common_utils.codex_model_catalog import (
|
||||
CODEX_BUNDLED_VERSION,
|
||||
CodexStock,
|
||||
CodexStockCatalog,
|
||||
CodexStockModel,
|
||||
bundled_codex_models,
|
||||
codex_base_instructions,
|
||||
uses_vendored_codex_catalog,
|
||||
)
|
||||
from litellm.types.llms.base_model import LiteLLMBaseModel
|
||||
|
||||
CODEX_CATALOG_OPENAI_URL: Final = "https://chatgpt.com/backend-api/codex/models"
|
||||
CODEX_CATALOG_BASE_URL: Final = "https://raw.githubusercontent.com/openai/codex"
|
||||
CODEX_CATALOG_RETRY_SECONDS: Final = 300.0
|
||||
CODEX_CATALOG_REFRESH_SECONDS: Final = 3600.0
|
||||
CODEX_CATALOG_FETCH_TIMEOUT_SECONDS: Final = 10.0
|
||||
CODEX_CATALOG_RESPONSE_DEADLINE_SECONDS: Final = 3.0
|
||||
CODEX_CATALOG_MAX_BYTES: Final = 16 * 1024 * 1024
|
||||
CODEX_CATALOG_CACHED_VERSIONS: Final = 16
|
||||
_RELEASE_VERSION: Final = re.compile(r"\d+\.\d+\.\d+(?:-[0-9A-Za-z.]+)?")
|
||||
_MODELS_PATH: Final = "codex-rs/models-manager/models.json"
|
||||
_PROMPT_PATH: Final = "codex-rs/models-manager/prompt.md"
|
||||
|
||||
_CacheSlot: TypeAlias = tuple[str, str | None]
|
||||
|
||||
|
||||
class _AsyncGetClient(Protocol):
|
||||
def get(
|
||||
self,
|
||||
url: str,
|
||||
*,
|
||||
headers: dict[str, str] | None = None, # mutable-ok: httpx's own get signature
|
||||
timeout: float | None = None,
|
||||
max_response_bytes: int | None = None,
|
||||
) -> Awaitable[httpx.Response]: ...
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _Cached:
|
||||
stock: CodexStock
|
||||
refetch_at: float | None
|
||||
from_openai: bool
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _Fetched:
|
||||
stock: CodexStock
|
||||
refetch_after: float | None
|
||||
from_openai: bool
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _FetchFailed:
|
||||
reason: str
|
||||
|
||||
|
||||
class _ModelMessages(LiteLLMBaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
instructions_template: str | None = None
|
||||
|
||||
|
||||
def _without_instructions_copy(model: CodexStockModel) -> CodexStockModel:
|
||||
"""`model` without the `base_instructions` OpenAI mirrors from its `model_messages.instructions_template`,
|
||||
the way openai/codex's own models.json ships it."""
|
||||
extra: Final[Mapping[str, object]] = model.model_extra or {}
|
||||
try:
|
||||
messages: Final = _ModelMessages.model_validate(extra.get("model_messages"))
|
||||
except ValidationError:
|
||||
return model
|
||||
return model.model_copy(update={"base_instructions": ""}) if messages.instructions_template else model
|
||||
|
||||
|
||||
def _over_vendored(models: tuple[CodexStockModel, ...]) -> Mapping[str, CodexStockModel]:
|
||||
return MappingProxyType({**bundled_codex_models(), **{model.slug: model for model in models}})
|
||||
|
||||
|
||||
def _default_client() -> _AsyncGetClient:
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] # its `params: dict` is untyped
|
||||
)
|
||||
from litellm.types.llms.custom_http import httpxSpecialProvider
|
||||
|
||||
return get_async_httpx_client(llm_provider=httpxSpecialProvider.CodexModelCatalog)
|
||||
|
||||
|
||||
_VENDORED_SOURCE: Final = f"vendored {CODEX_BUNDLED_VERSION}"
|
||||
|
||||
|
||||
def vendored_codex_stock() -> CodexStock:
|
||||
"""The pair vendored with LiteLLM, served when the requesting release's own cannot be."""
|
||||
return CodexStock(models=bundled_codex_models(), instructions=codex_base_instructions(), source=_VENDORED_SOURCE)
|
||||
|
||||
|
||||
def codex_release_tag(client_version: str | None) -> str | None:
|
||||
"""The openai/codex tag the Codex sending `client_version` was built from, None when the value is no
|
||||
release version. Codex sends its version without a pre-release suffix, so an alpha build of 0.167.0 asks
|
||||
for `rust-v0.167.0`, which has no tag until that release ships."""
|
||||
if client_version is None or _RELEASE_VERSION.fullmatch(client_version) is None:
|
||||
return None
|
||||
return f"rust-v{client_version}"
|
||||
|
||||
|
||||
class CodexCatalogSource:
|
||||
"""Resolves the stock catalog for a `client_version`, one fetch per release tag, concurrent requests
|
||||
for the same tag sharing that fetch. `client` is the HTTP client the fetch goes through (LiteLLM's
|
||||
shared async client when None), `clock` the monotonic clock that times retries and refreshes."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
client: _AsyncGetClient | None = None,
|
||||
base_url: str | None = None,
|
||||
openai_url: str | None = None,
|
||||
clock: Callable[[], float] = time.monotonic,
|
||||
retry_seconds: float = CODEX_CATALOG_RETRY_SECONDS,
|
||||
refresh_seconds: float = CODEX_CATALOG_REFRESH_SECONDS,
|
||||
timeout: float = CODEX_CATALOG_FETCH_TIMEOUT_SECONDS,
|
||||
response_deadline: float = CODEX_CATALOG_RESPONSE_DEADLINE_SECONDS,
|
||||
cached_versions: int = CODEX_CATALOG_CACHED_VERSIONS,
|
||||
) -> None:
|
||||
self._client = client
|
||||
self._base_url = (
|
||||
base_url if base_url is not None else os.getenv("LITELLM_CODEX_CATALOG_BASE_URL", CODEX_CATALOG_BASE_URL)
|
||||
).rstrip("/")
|
||||
self._openai_url = (
|
||||
openai_url
|
||||
if openai_url is not None
|
||||
else os.getenv("LITELLM_CODEX_CATALOG_OPENAI_URL", CODEX_CATALOG_OPENAI_URL)
|
||||
)
|
||||
self._clock = clock
|
||||
self._retry_seconds = retry_seconds
|
||||
self._refresh_seconds = refresh_seconds
|
||||
self._timeout = timeout
|
||||
self._response_deadline = response_deadline
|
||||
self._cached_versions = cached_versions
|
||||
self._cache: OrderedDict[_CacheSlot, _Cached] = OrderedDict() # mutable-ok: filled as releases are asked for
|
||||
self._pending: dict[_CacheSlot, asyncio.Task[CodexStock]] = {} # mutable-ok: the fetches in flight
|
||||
self._fetch_starts: tuple[float, ...] = ()
|
||||
|
||||
async def stock(self, client_version: str | None, *, openai_api_key: str | None = None) -> CodexStock:
|
||||
"""The catalog for ``client_version``: OpenAI's live one when an OpenAI key is at hand,
|
||||
else the release's models.json on GitHub, else the bundled pair."""
|
||||
tag: Final = codex_release_tag(client_version)
|
||||
if tag is None or client_version is None or uses_vendored_codex_catalog():
|
||||
return vendored_codex_stock()
|
||||
slot: Final = (tag, openai_api_key)
|
||||
cached: Final = self._cache.get(slot)
|
||||
if cached is not None:
|
||||
self._cache.move_to_end(slot)
|
||||
if cached.refetch_at is not None and cached.refetch_at <= self._clock():
|
||||
self._fetching(slot, tag, client_version, openai_api_key)
|
||||
return cached.stock
|
||||
fetch: Final = self._fetching(slot, tag, client_version, openai_api_key)
|
||||
return vendored_codex_stock() if fetch is None else await self._within_deadline(tag, fetch)
|
||||
|
||||
def _fetching(
|
||||
self, slot: _CacheSlot, tag: str, version: str, openai_api_key: str | None
|
||||
) -> asyncio.Task[CodexStock] | None:
|
||||
pending: Final = self._pending.get(slot)
|
||||
if pending is not None:
|
||||
return pending
|
||||
now: Final = self._clock()
|
||||
recent: Final = tuple(start for start in self._fetch_starts if start > now - self._retry_seconds)
|
||||
if len(recent) >= self._cached_versions:
|
||||
verbose_proxy_logger.warning(
|
||||
"Codex catalog %s not fetched, %d fetches started in the last %.0fs, serving the %s one",
|
||||
tag,
|
||||
len(recent),
|
||||
self._retry_seconds,
|
||||
vendored_codex_stock().source,
|
||||
)
|
||||
self._fetch_starts = recent
|
||||
return None
|
||||
self._fetch_starts = (*recent, now)
|
||||
task: Final = asyncio.ensure_future(self._resolve(slot, tag, version, openai_api_key))
|
||||
self._pending[slot] = task
|
||||
task.add_done_callback(lambda _: self._pending.pop(slot, None))
|
||||
return task
|
||||
|
||||
async def _within_deadline(self, tag: str, fetch: asyncio.Task[CodexStock]) -> CodexStock:
|
||||
done, _ = await asyncio.wait((fetch,), timeout=self._response_deadline)
|
||||
if fetch in done:
|
||||
return fetch.result()
|
||||
verbose_proxy_logger.warning(
|
||||
"Codex catalog %s still fetching after %.0fs, serving the %s one to this request",
|
||||
tag,
|
||||
self._response_deadline,
|
||||
vendored_codex_stock().source,
|
||||
)
|
||||
return vendored_codex_stock()
|
||||
|
||||
async def _resolve(self, slot: _CacheSlot, tag: str, version: str, openai_api_key: str | None) -> CodexStock:
|
||||
previous: Final = self._cache.get(slot)
|
||||
holds_openai: Final = previous is not None and previous.from_openai
|
||||
fetched: Final = await self._fetch(tag, version, openai_api_key, github_fallback=not holds_openai)
|
||||
now: Final = self._clock()
|
||||
entry: Final = (
|
||||
_Cached(
|
||||
stock=previous.stock if previous is not None else vendored_codex_stock(),
|
||||
refetch_at=now + self._retry_seconds,
|
||||
from_openai=holds_openai,
|
||||
)
|
||||
if isinstance(fetched, _FetchFailed)
|
||||
else _Cached(
|
||||
stock=fetched.stock,
|
||||
refetch_at=None if fetched.refetch_after is None else now + fetched.refetch_after,
|
||||
from_openai=fetched.from_openai,
|
||||
)
|
||||
)
|
||||
if isinstance(fetched, _FetchFailed):
|
||||
verbose_proxy_logger.warning(
|
||||
"Codex catalog %s not fetched (%s), serving the %s one for the next %.0fs",
|
||||
tag,
|
||||
fetched.reason,
|
||||
entry.stock.source,
|
||||
self._retry_seconds,
|
||||
)
|
||||
else:
|
||||
verbose_proxy_logger.info(
|
||||
"Codex catalog %s fetched from %s, %d models", tag, entry.stock.source, len(entry.stock.models)
|
||||
)
|
||||
self._cache[slot] = entry
|
||||
self._cache.move_to_end(slot)
|
||||
while len(self._cache) > self._cached_versions:
|
||||
del self._cache[self._eviction_victim(keep=slot)]
|
||||
return entry.stock
|
||||
|
||||
def _eviction_victim(self, keep: _CacheSlot) -> _CacheSlot:
|
||||
"""The least recently asked release other than ``keep`` still on the vendored pair, else the least recently
|
||||
asked one, so releases no catalog answered push each other out before a fetched release goes, while the
|
||||
release just asked keeps its retry time."""
|
||||
vendored: Final = (
|
||||
slot for slot, cached in self._cache.items() if slot != keep and cached.stock.source == _VENDORED_SOURCE
|
||||
)
|
||||
return next(vendored, next(iter(self._cache)))
|
||||
|
||||
async def _fetch(
|
||||
self, tag: str, version: str, openai_api_key: str | None, *, github_fallback: bool
|
||||
) -> _Fetched | _FetchFailed:
|
||||
if not openai_api_key or not self._openai_url:
|
||||
github: Final = await self._fetch_github(tag)
|
||||
return (
|
||||
github
|
||||
if isinstance(github, _FetchFailed)
|
||||
else _Fetched(stock=github, refetch_after=None, from_openai=False)
|
||||
)
|
||||
openai: Final = await self._fetch_openai(version, openai_api_key)
|
||||
if not isinstance(openai, _FetchFailed):
|
||||
return _Fetched(stock=openai, refetch_after=self._refresh_seconds, from_openai=True)
|
||||
if not github_fallback:
|
||||
return openai
|
||||
verbose_proxy_logger.info("Codex catalog %s not fetched from OpenAI (%s), trying GitHub", tag, openai.reason)
|
||||
fallback: Final = await self._fetch_github(tag)
|
||||
return (
|
||||
fallback
|
||||
if isinstance(fallback, _FetchFailed)
|
||||
else _Fetched(stock=fallback, refetch_after=self._retry_seconds, from_openai=False)
|
||||
)
|
||||
|
||||
async def _fetch_openai(self, version: str, openai_api_key: str) -> CodexStock | _FetchFailed:
|
||||
url: Final = f"{self._openai_url}?client_version={version}"
|
||||
body: Final = await self._get(url, headers={"Authorization": f"Bearer {openai_api_key}"})
|
||||
if isinstance(body, _FetchFailed):
|
||||
return body
|
||||
try:
|
||||
catalog: Final = CodexStockCatalog.model_validate_json(body)
|
||||
except ValidationError as error:
|
||||
return _FetchFailed(reason=f"{url} body does not parse: {type(error).__name__}")
|
||||
if not catalog.models:
|
||||
return _FetchFailed(reason=f"{url} catalog is empty")
|
||||
return CodexStock(
|
||||
models=_over_vendored(tuple(map(_without_instructions_copy, catalog.models))),
|
||||
instructions=codex_base_instructions(),
|
||||
source=f"openai {version}",
|
||||
)
|
||||
|
||||
async def _fetch_github(self, tag: str) -> CodexStock | _FetchFailed:
|
||||
models, prompt = await asyncio.gather(
|
||||
self._get(f"{self._base_url}/{tag}/{_MODELS_PATH}"), self._get(f"{self._base_url}/{tag}/{_PROMPT_PATH}")
|
||||
)
|
||||
if isinstance(models, _FetchFailed):
|
||||
return models
|
||||
if isinstance(prompt, _FetchFailed):
|
||||
return prompt
|
||||
try:
|
||||
catalog: Final = CodexStockCatalog.model_validate_json(models)
|
||||
instructions: Final = prompt.decode("utf-8")
|
||||
except (ValidationError, UnicodeDecodeError) as error:
|
||||
return _FetchFailed(reason=f"{tag} body does not parse: {type(error).__name__}")
|
||||
if not catalog.models or not instructions.strip():
|
||||
return _FetchFailed(reason=f"{tag} catalog or prompt is empty")
|
||||
return CodexStock(
|
||||
models=_over_vendored(catalog.models),
|
||||
instructions=instructions,
|
||||
source=f"github {tag}",
|
||||
)
|
||||
|
||||
async def _get(
|
||||
self,
|
||||
url: str,
|
||||
headers: dict[str, str] | None = None, # mutable-ok: httpx's own get signature
|
||||
) -> bytes | _FetchFailed:
|
||||
client: Final = self._client if self._client is not None else _default_client()
|
||||
try:
|
||||
response: Final = await asyncio.wait_for(
|
||||
client.get(url, headers=headers, timeout=self._timeout, max_response_bytes=CODEX_CATALOG_MAX_BYTES),
|
||||
self._timeout,
|
||||
)
|
||||
except (httpx.HTTPError, httpx.InvalidURL, HTTPResponseLimitError, ValueError, asyncio.TimeoutError) as error:
|
||||
return _FetchFailed(reason=f"{type(error).__name__} fetching {url}")
|
||||
if response.status_code != 200:
|
||||
return _FetchFailed(reason=f"HTTP {response.status_code} from {url}")
|
||||
if len(response.content) > CODEX_CATALOG_MAX_BYTES:
|
||||
return _FetchFailed(reason=f"{url} is over {CODEX_CATALOG_MAX_BYTES} bytes")
|
||||
return response.content
|
||||
|
||||
|
||||
@cache
|
||||
def codex_catalog_source() -> CodexCatalogSource:
|
||||
"""The process-wide source `/v1/models` resolves its stock catalog through."""
|
||||
return CodexCatalogSource()
|
||||
|
|
@ -12,15 +12,18 @@ upstream. A ``model_info.max_input_tokens`` the operator sets replaces a known m
|
|||
A fallback entry offers the model group's ``supported_reasoning_efforts`` as its reasoning levels,
|
||||
trimmed to the ones the requesting Codex release can parse.
|
||||
|
||||
The stock entries are Codex 0.159.3's bundled catalog, vendored unchanged from
|
||||
https://github.com/openai/codex/blob/rust-v0.159.3/codex-rs/models-manager/models.json (Apache-2.0)
|
||||
as ``codex_bundled_models_0.159.3.json``, and ``codex_base_instructions.md`` is
|
||||
``codex-rs/models-manager/prompt.md`` at the same tag. Refresh both by downloading those two paths at
|
||||
a newer tag.
|
||||
The stock entries come from the requesting Codex release's own catalog, fetched by
|
||||
``codex_catalog_source.CodexCatalogSource`` and passed in as a ``CodexStock``, laid over the bundled entries
|
||||
below. When none is passed, or that release's cannot be fetched, they are Codex ``CODEX_BUNDLED_VERSION``'s
|
||||
bundled catalog, vendored unchanged from https://github.com/openai/codex/blob/rust-v<version>/codex-rs/models-manager/models.json
|
||||
(Apache-2.0) as ``codex_bundled_models_<version>.json``, with ``codex_base_instructions.md`` being
|
||||
``codex-rs/models-manager/prompt.md`` at the same tag. Refresh both by downloading those two paths at a
|
||||
newer tag and renaming the json after it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from functools import cache
|
||||
|
|
@ -28,11 +31,13 @@ from itertools import accumulate, pairwise
|
|||
from pathlib import Path
|
||||
from types import MappingProxyType, NoneType
|
||||
from typing import TYPE_CHECKING, Annotated, Final, Literal, TypeAlias
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from packaging.version import Version
|
||||
from pydantic import ConfigDict, Field, StringConstraints, TypeAdapter, ValidationError
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.litellm_core_utils.offline_mode import is_offline_mode
|
||||
from litellm.litellm_core_utils.reasoning_effort_strength import nearest_declared_reasoning_effort
|
||||
from litellm.types.llms.base_model import LiteLLMBaseModel
|
||||
|
||||
|
|
@ -40,9 +45,11 @@ if TYPE_CHECKING:
|
|||
from litellm.router import Router
|
||||
from litellm.router_api import RouterRegistry
|
||||
from litellm.types.proxy.model_listing import ModelInfoResponse
|
||||
from litellm.types.router import DeploymentTypedDict
|
||||
|
||||
CODEX_BASE_INSTRUCTIONS_PATH: Final = Path(__file__).with_name("codex_base_instructions.md")
|
||||
CODEX_BUNDLED_MODELS_PATH: Final = Path(__file__).with_name("codex_bundled_models_0.159.3.json")
|
||||
CODEX_BUNDLED_VERSION: Final = "0.166.0"
|
||||
CODEX_BUNDLED_MODELS_PATH: Final = Path(__file__).with_name(f"codex_bundled_models_{CODEX_BUNDLED_VERSION}.json")
|
||||
CODEX_CATALOG_BYTE_LIMIT: Final = 1024 * 1024
|
||||
CODEX_CHAT_MODES: Final = frozenset({"chat", "responses"})
|
||||
|
||||
|
|
@ -50,6 +57,7 @@ _TierId = Annotated[str, StringConstraints(strip_whitespace=True, min_length=1)]
|
|||
_BODY_PREFIX: Final = '{"models":['
|
||||
_BODY_SUFFIX: Final = "]}"
|
||||
_FIRST_CODEX_WITH_CUSTOM_EFFORTS: Final = Version("0.138.0")
|
||||
_FIRST_CODEX_WITHOUT_CATALOG_BYTE_LIMIT: Final = Version("0.166.0")
|
||||
_EFFORTS_EVERY_CODEX_PARSES: Final = frozenset({"none", "minimal", "low", "medium", "high", "xhigh"})
|
||||
_PREFERRED_DEFAULT_EFFORT: Final = "medium"
|
||||
_CODEX_UNSET_EFFORT: Final = "none"
|
||||
|
|
@ -161,6 +169,7 @@ class CodexStockModel(LiteLLMBaseModel):
|
|||
supported_reasoning_levels: tuple[CodexReasoningLevel, ...] = ()
|
||||
context_window: int | None = None
|
||||
max_context_window: int | None = None
|
||||
base_instructions: str = ""
|
||||
|
||||
|
||||
class CodexStockCatalog(LiteLLMBaseModel):
|
||||
|
|
@ -192,6 +201,16 @@ class CodexCatalogRow:
|
|||
configured_max_input_tokens: tuple[int | None, ...] = ()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CodexStock:
|
||||
"""One Codex release's own entries by slug and its base prompt; `source` names the release tag or the
|
||||
vendored version they came from."""
|
||||
|
||||
models: Mapping[str, CodexStockModel]
|
||||
instructions: str
|
||||
source: str
|
||||
|
||||
|
||||
@cache
|
||||
def bundled_codex_models() -> Mapping[str, CodexStockModel]:
|
||||
"""Codex's bundled catalog by slug, read once from the vendored file."""
|
||||
|
|
@ -299,6 +318,18 @@ def _parses_custom_efforts(client_version: str | None) -> bool:
|
|||
return False
|
||||
|
||||
|
||||
def codex_catalog_byte_limit(client_version: str | None) -> int | None:
|
||||
"""The cap the asking Codex puts on a `model_catalog_url` body: 1 MiB through 0.165, none from 0.166.0,
|
||||
which dropped it (openai/codex#56525)."""
|
||||
try:
|
||||
uncapped: Final = (
|
||||
client_version is not None and Version(client_version) >= _FIRST_CODEX_WITHOUT_CATALOG_BYTE_LIMIT
|
||||
)
|
||||
except ValueError:
|
||||
return CODEX_CATALOG_BYTE_LIMIT
|
||||
return None if uncapped else CODEX_CATALOG_BYTE_LIMIT
|
||||
|
||||
|
||||
def codex_reasoning_levels(
|
||||
efforts: Sequence[str], client_version: str | None
|
||||
) -> tuple[tuple[CodexFallbackReasoningLevel, ...], str | None]:
|
||||
|
|
@ -494,8 +525,8 @@ def codex_models_response_json(
|
|||
"""Codex's `ModelsResponse` body for `rows`, in listing order, cut to Codex's byte limit.
|
||||
|
||||
Only chat-capable rows are listed (Codex cannot drive an embedding or image model, and a
|
||||
wildcard id is no model), their order is their `priority`. Codex drops a catalog over its
|
||||
byte limit silently and keeps its bundled one, so the body holds only the entries that fit,
|
||||
wildcard id is no model), their order is their `priority`. Codex before 0.166.0 drops a catalog over
|
||||
its byte limit silently and keeps its bundled one, so the body holds only the entries that fit,
|
||||
the ones offering a service tier taken first and the rest in listing order, an entry too large
|
||||
for the bytes left passed over, every kept entry at its listing position; `byte_limit=None`
|
||||
keeps every entry. A row whose deployments set `model_info.max_input_tokens` gets the smallest
|
||||
|
|
@ -552,6 +583,135 @@ def _catalog_row(
|
|||
)
|
||||
|
||||
|
||||
OPENAI_API_HOST: Final = "api.openai.com"
|
||||
FEDRAMP_KEY_PREFIXES: Final = ("sk-proj-fedramp-", "sk-svcacct-fedramp-")
|
||||
|
||||
|
||||
_DEPLOYMENT_BASE_PARAMS: Final = ("api_base", "base_url")
|
||||
|
||||
|
||||
def _is_openai_host(url: object) -> bool:
|
||||
if not isinstance(url, str):
|
||||
return False
|
||||
try:
|
||||
host: Final = urlparse(url).hostname
|
||||
except ValueError:
|
||||
return False
|
||||
return host == OPENAI_API_HOST
|
||||
|
||||
|
||||
def _every_call_goes_through_a_litellm_proxy() -> bool:
|
||||
import litellm # noqa: PLC0415 # litellm imports this module while it is initializing
|
||||
|
||||
return litellm.LiteLLMProxyChatConfig.should_use_litellm_proxy_by_default() is True
|
||||
|
||||
|
||||
def _is_openai_origin(bases: Sequence[object]) -> bool:
|
||||
"""Whether a deployment's OpenAI requests go to OpenAI itself, so its key may go to OpenAI's catalog
|
||||
endpoint: every `api_base` and `base_url` the deployment and its credential set, else the global
|
||||
`litellm.api_base`, `OPENAI_BASE_URL` or `OPENAI_API_BASE` override the OpenAI handler would apply, else
|
||||
OpenAI."""
|
||||
import litellm # noqa: PLC0415 # litellm imports this module while it is initializing, so the global is read late
|
||||
|
||||
named: Final = tuple(filter(None, bases))
|
||||
if named:
|
||||
return all(map(_is_openai_host, named))
|
||||
fallback: Final = litellm.api_base or os.getenv("OPENAI_BASE_URL") or os.getenv("OPENAI_API_BASE")
|
||||
return not fallback or _is_openai_host(fallback)
|
||||
|
||||
|
||||
def _credential_values(name: object) -> Mapping[str, object]:
|
||||
if not isinstance(name, str) or not name:
|
||||
return MappingProxyType({})
|
||||
from litellm.litellm_core_utils.credential_accessor import (
|
||||
CredentialAccessor, # noqa: PLC0415 # credential_accessor imports litellm, which imports this module
|
||||
)
|
||||
|
||||
return MappingProxyType(CredentialAccessor.get_credential_values(name))
|
||||
|
||||
|
||||
def _deployment_provider(params: Mapping[str, object]) -> str:
|
||||
"""The provider litellm routes a deployment to: its `custom_llm_provider`, else its model's prefix, else
|
||||
`openai` for a bare OpenAI model name such as `gpt-5.5`."""
|
||||
import litellm # noqa: PLC0415 # litellm imports this module while it is initializing
|
||||
|
||||
custom: Final = params.get("custom_llm_provider")
|
||||
model: Final = params.get("model")
|
||||
if isinstance(custom, str) and custom:
|
||||
return custom
|
||||
if not isinstance(model, str):
|
||||
return ""
|
||||
prefix, slash, _ = model.partition("/")
|
||||
if slash:
|
||||
return prefix
|
||||
return "openai" if model in litellm.open_ai_chat_completion_models else ""
|
||||
|
||||
|
||||
def _deployment_openai_key(deployment: DeploymentTypedDict) -> str | None:
|
||||
"""The key a deployment sends OpenAI, its own values winning over its named credential's the way the call
|
||||
path fills only what the deployment lacks and ``OPENAI_API_KEY`` standing in when neither sets one, when every
|
||||
base either one sets is OpenAI's and the call does not go through a LiteLLM proxy."""
|
||||
params: Final = MappingProxyType[str, object](deployment["litellm_params"])
|
||||
credential: Final = _credential_values(params.get("litellm_credential_name"))
|
||||
effective: Final = MappingProxyType({**credential, **params})
|
||||
bases: Final = (
|
||||
*(params.get(key) for key in _DEPLOYMENT_BASE_PARAMS),
|
||||
*(credential.get(key) for key in _DEPLOYMENT_BASE_PARAMS),
|
||||
)
|
||||
if (
|
||||
effective.get("use_litellm_proxy") is True
|
||||
or _deployment_provider(effective) != "openai"
|
||||
or not _is_openai_origin(bases)
|
||||
):
|
||||
return None
|
||||
key: Final = _resolved_key(effective.get("api_key"))
|
||||
return None if key is None or key.startswith(FEDRAMP_KEY_PREFIXES) else key
|
||||
|
||||
|
||||
def _resolved_key(api_key: object) -> str | None:
|
||||
if not isinstance(api_key, str) or not api_key:
|
||||
return os.getenv("OPENAI_API_KEY") or None
|
||||
if api_key.startswith("oidc/"):
|
||||
return None
|
||||
from litellm.secret_managers.main import (
|
||||
get_secret_str, # noqa: PLC0415 # secret_managers.main imports litellm, which imports this module
|
||||
)
|
||||
|
||||
return (get_secret_str(api_key) if api_key.startswith("os.environ/") else api_key) or None
|
||||
|
||||
|
||||
def uses_vendored_codex_catalog() -> bool:
|
||||
return is_offline_mode() or os.getenv("LITELLM_LOCAL_CODEX_CATALOG", "").strip().lower() == "true"
|
||||
|
||||
|
||||
def codex_catalog_openai_key(
|
||||
llm_router: Router | None,
|
||||
entries: Sequence[tuple[str, str]],
|
||||
team_id: str | None = None,
|
||||
) -> str | None:
|
||||
"""The OpenAI API key the catalog fetch authenticates with: the first one among the listed entries'
|
||||
deployments that routes to OpenAI with every `api_base` and `base_url` it or its credential sets being
|
||||
OpenAI's own, ``OPENAI_API_KEY`` from the environment for one that sets no key. A key for an
|
||||
OpenAI-compatible server or a FedRAMP key (Codex sends those to ``gov.chatgpt.com``) never leaves for
|
||||
OpenAI's commercial catalog, a listing without an OpenAI deployment sends none, and no key is looked up
|
||||
while the catalog stays local or calls go through a LiteLLM proxy."""
|
||||
if uses_vendored_codex_catalog() or _every_call_goes_through_a_litellm_proxy():
|
||||
return None
|
||||
registry: Final = llm_router.registry() if llm_router is not None else None
|
||||
deployments: Final = (
|
||||
tuple(
|
||||
deployment
|
||||
for _, lookup_id in entries
|
||||
for deployment in registry.deployments(
|
||||
lookup_id, source="all", team_id=team_id
|
||||
) # comprehension-ok: flattening the entries' deployment lists
|
||||
)
|
||||
if registry is not None
|
||||
else ()
|
||||
)
|
||||
return next(filter(None, map(_deployment_openai_key, deployments)), None)
|
||||
|
||||
|
||||
def codex_catalog_rows(
|
||||
rows: Sequence[ModelInfoResponse],
|
||||
entries: Sequence[tuple[str, str]],
|
||||
|
|
@ -583,10 +743,16 @@ def codex_model_list_body(
|
|||
label_entries: Sequence[tuple[str, str]] | None = None,
|
||||
*,
|
||||
client_version: str | None = None,
|
||||
stock: CodexStock | None = None,
|
||||
) -> str:
|
||||
"""The `/v1/models?client_version=...` body, logging the models Codex's byte limit left out."""
|
||||
"""The `/v1/models?client_version=...` body built on `stock` (the vendored pair when None), logging the
|
||||
models Codex's byte limit left out."""
|
||||
body: Final = codex_models_response_json(
|
||||
codex_catalog_rows(rows, entries, llm_router, team_id, label_entries), client_version=client_version
|
||||
codex_catalog_rows(rows, entries, llm_router, team_id, label_entries),
|
||||
stock=stock.models if stock is not None else None,
|
||||
instructions=stock.instructions if stock is not None else None,
|
||||
byte_limit=codex_catalog_byte_limit(client_version),
|
||||
client_version=client_version,
|
||||
)
|
||||
if body.left_out:
|
||||
verbose_proxy_logger.warning(
|
||||
|
|
|
|||
|
|
@ -518,7 +518,8 @@ from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import (
|
|||
AuthCacheInvalidationSubscriber,
|
||||
)
|
||||
from litellm.proxy.common_utils.callback_utils import initialize_callbacks_on_proxy
|
||||
from litellm.proxy.common_utils.codex_model_catalog import codex_model_list_body
|
||||
from litellm.proxy.common_utils.codex_catalog_source import codex_catalog_source
|
||||
from litellm.proxy.common_utils.codex_model_catalog import codex_catalog_openai_key, codex_model_list_body
|
||||
from litellm.proxy.common_utils.config_includes import resolve_include_file_path, resolve_includes
|
||||
from litellm.proxy.common_utils.config_sync_pubsub import ConfigSyncSubscriber
|
||||
from litellm.proxy.common_utils.credential_hydration import decrypted_or_stored
|
||||
|
|
@ -12843,6 +12844,12 @@ async def model_list(
|
|||
llm_router,
|
||||
team_id or user_api_key_dict.team_id,
|
||||
client_version=client_version,
|
||||
stock=await codex_catalog_source().stock(
|
||||
client_version,
|
||||
openai_api_key=codex_catalog_openai_key(
|
||||
llm_router, admin_entries, team_id or user_api_key_dict.team_id
|
||||
),
|
||||
),
|
||||
),
|
||||
media_type="application/json",
|
||||
)
|
||||
|
|
@ -12925,6 +12932,10 @@ async def model_list(
|
|||
team_id or user_api_key_dict.team_id,
|
||||
labels,
|
||||
client_version=client_version,
|
||||
stock=await codex_catalog_source().stock(
|
||||
client_version,
|
||||
openai_api_key=codex_catalog_openai_key(llm_router, entries, team_id or user_api_key_dict.team_id),
|
||||
),
|
||||
),
|
||||
media_type="application/json",
|
||||
)
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ class httpxSpecialProvider(str, Enum):
|
|||
ROICalculator = "roi_calculator"
|
||||
Sandbox = "sandbox"
|
||||
ModelCostMap = "model_cost_map"
|
||||
CodexModelCatalog = "codex_model_catalog"
|
||||
PasswordBreachCheck = "password_breach_check"
|
||||
ASGI = "asgi"
|
||||
AgentHarness = "agent_harness"
|
||||
|
|
|
|||
|
|
@ -61,7 +61,7 @@ include = [
|
|||
"litellm/router_strategy/complexity_router/fuse_presets.json",
|
||||
"litellm/proxy/model_insights_tasks.json",
|
||||
"litellm/proxy/common_utils/codex_base_instructions.md",
|
||||
"litellm/proxy/common_utils/codex_bundled_models_0.159.3.json",
|
||||
"litellm/proxy/common_utils/codex_bundled_models_0.166.0.json",
|
||||
]
|
||||
exclude = [
|
||||
"litellm/proxy/_experimental/out",
|
||||
|
|
|
|||
|
|
@ -340,7 +340,7 @@ include = [
|
|||
"litellm/proxy/model_insights_tasks.json",
|
||||
"litellm/proxy/public_endpoints/whats_new.json",
|
||||
"litellm/proxy/common_utils/codex_base_instructions.md",
|
||||
"litellm/proxy/common_utils/codex_bundled_models_0.159.3.json",
|
||||
"litellm/proxy/common_utils/codex_bundled_models_0.166.0.json",
|
||||
{ path = "model_prices_and_context_window.json", format = "sdist" },
|
||||
]
|
||||
exclude = [
|
||||
|
|
|
|||
193
tests/integration/compatibility/test_codex_catalog_fetch.py
Normal file
193
tests/integration/compatibility/test_codex_catalog_fetch.py
Normal file
|
|
@ -0,0 +1,193 @@
|
|||
"""`/v1/models?client_version=` serves the requesting Codex release's own catalog, fetched from the
|
||||
openai/codex raw tree (a wire peer standing in for raw.githubusercontent.com here), and the vendored pair
|
||||
for a release that tree has no tag for."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import uuid
|
||||
from collections.abc import Iterator
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
from pydantic import JsonValue
|
||||
|
||||
from tests.integration._support.client import (
|
||||
JSON_OBJECT,
|
||||
Gateway,
|
||||
eventually,
|
||||
gateway_from_environment,
|
||||
object_value,
|
||||
string_value,
|
||||
)
|
||||
from tests.integration._support.process import owned_proxy
|
||||
from tests.integration._support.wire import Reply, Request, Wire, wire_server
|
||||
|
||||
pytestmark: Final = pytest.mark.shares_worker_state
|
||||
|
||||
ROOT: Final = Path(os.environ.get("INTEGRATION_PROXY_ROOT") or Path(__file__).resolve().parents[3])
|
||||
BUNDLED_STOCK_PATH: Final = ROOT / "litellm" / "proxy" / "common_utils" / "codex_bundled_models_0.166.0.json"
|
||||
BASE_INSTRUCTIONS_PATH: Final = ROOT / "litellm" / "proxy" / "common_utils" / "codex_base_instructions.md"
|
||||
PUBLISHED_VERSION: Final = "0.999.0"
|
||||
UNPUBLISHED_VERSION: Final = "0.998.0"
|
||||
OPENAI_VERSION: Final = "0.997.0"
|
||||
OPENAI_SLUG: Final = "gpt-openai-release"
|
||||
OPENAI_PROMPT: Final = "# Base prompt OpenAI serves for its release"
|
||||
OPENAI_MODELS_PATH: Final = "/codex/models"
|
||||
DEPLOYMENT_KEY: Final = "integration-provider-key"
|
||||
FRESH_SLUG: Final = "gpt-fresh-release"
|
||||
FRESH_PROMPT: Final = "# Base prompt of the published release"
|
||||
MODELS_PATH: Final = "codex-rs/models-manager/models.json"
|
||||
PROMPT_PATH: Final = "codex-rs/models-manager/prompt.md"
|
||||
FRESH_CONNECTION: Final = {"Connection": "close"}
|
||||
CONVERGENCE_SECONDS: Final = 60
|
||||
|
||||
|
||||
def _vendored_models() -> tuple[dict[str, JsonValue], ...]:
|
||||
models: Final = JSON_OBJECT.validate_json(BUNDLED_STOCK_PATH.read_bytes())["models"]
|
||||
assert isinstance(models, list)
|
||||
return tuple(object_value(model) for model in models)
|
||||
|
||||
|
||||
def _vendored_slugs() -> frozenset[JsonValue]:
|
||||
return frozenset(model["slug"] for model in _vendored_models())
|
||||
|
||||
|
||||
def _vendored_entry(slug: str) -> dict[str, JsonValue]:
|
||||
return next(model for model in _vendored_models() if model["slug"] == slug)
|
||||
|
||||
|
||||
def _published_entry() -> dict[str, JsonValue]:
|
||||
return {**_vendored_entry("gpt-5.5"), "slug": FRESH_SLUG, "display_name": "Fresh Release", "upgrade": None}
|
||||
|
||||
|
||||
def _openai_entry() -> dict[str, JsonValue]:
|
||||
published: Final = _published_entry()
|
||||
return {
|
||||
**published,
|
||||
"slug": OPENAI_SLUG,
|
||||
"display_name": "OpenAI Release",
|
||||
"base_instructions": OPENAI_PROMPT,
|
||||
"model_messages": {**object_value(published["model_messages"]), "instructions_template": OPENAI_PROMPT},
|
||||
}
|
||||
|
||||
|
||||
def _raw_tree(request: Request) -> Reply:
|
||||
if request.target.startswith(f"{OPENAI_MODELS_PATH}?"):
|
||||
served: Final = request.target == f"{OPENAI_MODELS_PATH}?client_version={OPENAI_VERSION}"
|
||||
if served and request.headers.get("authorization") == f"Bearer {DEPLOYMENT_KEY}":
|
||||
return Reply(body=json.dumps({"models": [_openai_entry()]}).encode())
|
||||
return Reply(status=401, body=b'{"detail":"Unauthorized"}')
|
||||
if request.target == f"/rust-v{PUBLISHED_VERSION}/{MODELS_PATH}":
|
||||
return Reply(body=json.dumps({"models": [_published_entry()]}).encode())
|
||||
if request.target == f"/rust-v{PUBLISHED_VERSION}/{PROMPT_PATH}":
|
||||
return Reply(body=FRESH_PROMPT.encode(), content_type="text/plain; charset=utf-8")
|
||||
return Reply(status=404, body=b"404: Not Found", content_type="text/plain; charset=utf-8")
|
||||
|
||||
|
||||
def _config_without_model_list(directory: Path) -> Path:
|
||||
"""The shared config minus its models, so the catalog's 1 MiB cut never drops the models a test adds."""
|
||||
shared_config: Final = object_value(yaml.safe_load((ROOT / "tests/integration/proxy_config.yaml").read_text()))
|
||||
path: Final = directory / "proxy_config_without_model_list.yaml"
|
||||
path.write_text(yaml.safe_dump({name: value for name, value in shared_config.items() if name != "model_list"}))
|
||||
return path
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def raw_tree() -> Iterator[Wire]:
|
||||
with wire_server(_raw_tree) as wire:
|
||||
yield wire
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def gateway(tmp_path_factory: pytest.TempPathFactory, raw_tree: Wire) -> Iterator[Gateway]:
|
||||
with gateway_from_environment() as parent_gateway:
|
||||
directory: Final = tmp_path_factory.mktemp("codex_catalog_fetch")
|
||||
overrides: Final = {
|
||||
"LITELLM_LOCAL_CODEX_CATALOG": "False",
|
||||
"LITELLM_CODEX_CATALOG_BASE_URL": raw_tree.url,
|
||||
"LITELLM_CODEX_CATALOG_OPENAI_URL": f"{raw_tree.url}{OPENAI_MODELS_PATH}",
|
||||
}
|
||||
config: Final = _config_without_model_list(directory)
|
||||
with owned_proxy(parent_gateway, directory, overrides, config=config) as isolated_gateway:
|
||||
yield isolated_gateway
|
||||
|
||||
|
||||
def _catalog_listing(gateway: Gateway, client_version: str, *names: str) -> dict[str, dict[str, JsonValue]]:
|
||||
"""The catalog once every model in `names` is listed, since a `/model/new` reaches the listing asynchronously."""
|
||||
return eventually(
|
||||
lambda: _catalog(gateway, client_version),
|
||||
lambda value: all(name in value for name in names),
|
||||
seconds=CONVERGENCE_SECONDS,
|
||||
)
|
||||
|
||||
|
||||
def _catalog(gateway: Gateway, client_version: str) -> dict[str, dict[str, JsonValue]]:
|
||||
response: Final = gateway.request(
|
||||
"GET", "/v1/models", params={"client_version": client_version}, headers=FRESH_CONNECTION
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
models: Final = JSON_OBJECT.validate_json(response.content)["models"]
|
||||
assert isinstance(models, list), response.text
|
||||
return {string_value(object_value(entry)["slug"]): object_value(entry) for entry in models}
|
||||
|
||||
|
||||
def test_a_release_openai_serves_is_fetched_from_openai_with_the_deployment_key_and_github_is_skipped(
|
||||
gateway: Gateway, raw_tree: Wire
|
||||
) -> None:
|
||||
assert OPENAI_SLUG not in _vendored_slugs()
|
||||
with gateway.scenario() as scenario:
|
||||
scenario.model(name=OPENAI_SLUG, api_base="https://api.openai.com/v1")
|
||||
listing: Final = _catalog_listing(gateway, OPENAI_VERSION, OPENAI_SLUG)
|
||||
expected: Final = {
|
||||
**_openai_entry(),
|
||||
"base_instructions": "",
|
||||
"priority": listing[OPENAI_SLUG]["priority"],
|
||||
"visibility": "list",
|
||||
"supported_in_api": True,
|
||||
"upgrade": None,
|
||||
}
|
||||
assert listing[OPENAI_SLUG] == expected, listing[OPENAI_SLUG]
|
||||
requests: Final = raw_tree.drain()
|
||||
openai_requests: Final = [request for request in requests if request.target.startswith(OPENAI_MODELS_PATH)]
|
||||
assert len(openai_requests) == gateway.workers, [request.target for request in requests]
|
||||
assert {request.headers.get("authorization") for request in openai_requests} == {f"Bearer {DEPLOYMENT_KEY}"}
|
||||
assert not any(request.target.startswith(f"/rust-v{OPENAI_VERSION}/") for request in requests), requests
|
||||
|
||||
|
||||
def test_a_published_release_gets_its_own_entry_for_a_model_the_vendored_pair_lacks(
|
||||
gateway: Gateway, raw_tree: Wire
|
||||
) -> None:
|
||||
assert FRESH_SLUG not in _vendored_slugs()
|
||||
with gateway.scenario() as scenario:
|
||||
scenario.model(name=FRESH_SLUG)
|
||||
first: Final = _catalog_listing(gateway, PUBLISHED_VERSION, FRESH_SLUG)
|
||||
second: Final = _catalog(gateway, PUBLISHED_VERSION)
|
||||
published: Final = _published_entry()
|
||||
expected: Final = {
|
||||
**published,
|
||||
"priority": first[FRESH_SLUG]["priority"],
|
||||
"visibility": "list",
|
||||
"supported_in_api": True,
|
||||
"upgrade": None,
|
||||
}
|
||||
assert first[FRESH_SLUG] == expected, first[FRESH_SLUG]
|
||||
assert second[FRESH_SLUG] == expected, second[FRESH_SLUG]
|
||||
targets: Final = [request.target for request in raw_tree.drain()]
|
||||
assert targets.count(f"/rust-v{PUBLISHED_VERSION}/{MODELS_PATH}") == gateway.workers, targets
|
||||
assert targets.count(f"/rust-v{PUBLISHED_VERSION}/{PROMPT_PATH}") == gateway.workers, targets
|
||||
|
||||
|
||||
def test_an_unpublished_release_gets_the_vendored_pair(gateway: Gateway, raw_tree: Wire) -> None:
|
||||
with gateway.scenario() as scenario:
|
||||
unknown_name: Final = scenario.model()
|
||||
stock_name: Final = scenario.model(name=f"codex-stock-{uuid.uuid4().hex}", model="openai/gpt-5.5")
|
||||
catalog: Final = _catalog_listing(gateway, UNPUBLISHED_VERSION, unknown_name, stock_name)
|
||||
fresh: Final = catalog[unknown_name]
|
||||
assert fresh["shell_type"] == "unified_exec" and fresh["apply_patch_tool_type"] is None, fresh
|
||||
assert fresh["base_instructions"] == BASE_INSTRUCTIONS_PATH.read_text(encoding="utf-8"), fresh
|
||||
vendored: Final = _vendored_entry("gpt-5.5")
|
||||
assert catalog[stock_name]["shell_type"] == vendored["shell_type"], catalog[stock_name]
|
||||
assert catalog[stock_name]["model_messages"] == vendored["model_messages"], catalog[stock_name]
|
||||
assert f"/rust-v{UNPUBLISHED_VERSION}/{MODELS_PATH}" in [request.target for request in raw_tree.drain()]
|
||||
|
|
@ -43,7 +43,7 @@ RELOAD_MARGIN_SECONDS: Final = 5.0
|
|||
PROXY_WORKERS: Final = int(os.environ.get("INTEGRATION_PROXY_WORKERS", "1"))
|
||||
WORKER_SYNC_SECONDS: Final = 0.0 if PROXY_WORKERS == 1 else PROXY_CONFIG_RELOAD_INTERVAL_SECONDS + RELOAD_MARGIN_SECONDS
|
||||
ROOT: Final = Path(os.environ.get("INTEGRATION_PROXY_ROOT") or Path(__file__).resolve().parents[3])
|
||||
BUNDLED_STOCK_PATH: Final = ROOT / "litellm" / "proxy" / "common_utils" / "codex_bundled_models_0.159.3.json"
|
||||
BUNDLED_STOCK_PATH: Final = ROOT / "litellm" / "proxy" / "common_utils" / "codex_bundled_models_0.166.0.json"
|
||||
BASE_INSTRUCTIONS_PATH: Final = ROOT / "litellm" / "proxy" / "common_utils" / "codex_base_instructions.md"
|
||||
FRESH_CONNECTION: Final = {"Connection": "close"}
|
||||
ANTHROPIC_HEADERS: Final = {"anthropic-version": "2023-06-01"}
|
||||
|
|
|
|||
|
|
@ -130,7 +130,7 @@ def test_yaml_tiers_and_alias_in_owned_proxy(gateway: Gateway, tmp_path: Path) -
|
|||
|
||||
|
||||
ROOT: Final = Path(os.environ.get("INTEGRATION_PROXY_ROOT") or Path(__file__).resolve().parents[3])
|
||||
BUNDLED_STOCK_PATH: Final = ROOT / "litellm" / "proxy" / "common_utils" / "codex_bundled_models_0.159.3.json"
|
||||
BUNDLED_STOCK_PATH: Final = ROOT / "litellm" / "proxy" / "common_utils" / "codex_bundled_models_0.166.0.json"
|
||||
STOCK_UPSTREAM: Final = "gpt-5.5"
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -125,6 +125,7 @@ def main() -> int:
|
|||
"PYTHONPATH": os.pathsep.join((str(root), str(root / "tests"), str(root / "tests/e2e"))),
|
||||
"INTEGRATION_RESULTS_DIR": str(output),
|
||||
"LITELLM_LOCAL_MODEL_COST_MAP": "True",
|
||||
"LITELLM_LOCAL_CODEX_CATALOG": "True",
|
||||
"LITELLM_MODE": "PRODUCTION",
|
||||
"PYTHON_DOTENV_DISABLED": "1",
|
||||
**worker_settings(options.group, options.workers),
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ HOST_ENVIRONMENT_ALLOWLIST: Final = frozenset(
|
|||
"TZ",
|
||||
"VIRTUAL_ENV",
|
||||
"LITELLM_LOCAL_MODEL_COST_MAP",
|
||||
"LITELLM_LOCAL_CODEX_CATALOG",
|
||||
"TIKTOKEN_CACHE_DIR",
|
||||
)
|
||||
)
|
||||
|
|
@ -42,6 +43,7 @@ HOST_ONLY_ENVIRONMENT: Final = frozenset(
|
|||
|
||||
os.environ["PYTHON_DOTENV_DISABLED"] = "1"
|
||||
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
|
||||
os.environ["LITELLM_LOCAL_CODEX_CATALOG"] = "True" # test-quality-ok: set once so no unit test fetches github
|
||||
|
||||
import litellm # noqa: E402 # litellm reads LITELLM_LOCAL_MODEL_COST_MAP at import
|
||||
import litellm.router as litellm_router_module # noqa: E402 # same import-time dependency
|
||||
|
|
|
|||
653
tests/unit/proxy/common_utils/test_codex_catalog_source.py
Normal file
653
tests/unit/proxy/common_utils/test_codex_catalog_source.py
Normal file
|
|
@ -0,0 +1,653 @@
|
|||
import asyncio
|
||||
import json
|
||||
from collections import ChainMap
|
||||
from collections.abc import Awaitable, Callable, Mapping
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPResponseLimitError
|
||||
from litellm.proxy.common_utils.codex_catalog_source import (
|
||||
CODEX_CATALOG_BASE_URL,
|
||||
CODEX_CATALOG_MAX_BYTES,
|
||||
CODEX_CATALOG_OPENAI_URL,
|
||||
CodexCatalogSource,
|
||||
codex_release_tag,
|
||||
vendored_codex_stock,
|
||||
)
|
||||
from litellm.proxy.common_utils.codex_model_catalog import (
|
||||
CodexCatalogRow,
|
||||
CodexStockModel,
|
||||
codex_base_instructions,
|
||||
codex_models_response_json,
|
||||
)
|
||||
|
||||
JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
|
||||
RELEASE: Final = "0.166.0"
|
||||
FRESH_SLUG: Final = "gpt-fresh"
|
||||
FRESH_ENTRY: Final = {
|
||||
"slug": FRESH_SLUG,
|
||||
"display_name": "Fresh",
|
||||
"priority": 0,
|
||||
"visibility": "list",
|
||||
"shell_type": "shell_command",
|
||||
"apply_patch_tool_type": "freeform",
|
||||
"supports_parallel_tool_calls": True,
|
||||
}
|
||||
FRESH_CATALOG: Final = json.dumps({"models": [FRESH_ENTRY]}).encode()
|
||||
FRESH_PROMPT: Final = b"# Fresh base prompt"
|
||||
|
||||
|
||||
def _tree(version: str, catalog: bytes = FRESH_CATALOG, prompt: bytes = FRESH_PROMPT) -> dict[str, tuple[int, bytes]]:
|
||||
base: Final = f"{CODEX_CATALOG_BASE_URL}/rust-v{version}/codex-rs/models-manager"
|
||||
return {f"{base}/models.json": (200, catalog), f"{base}/prompt.md": (200, prompt)}
|
||||
|
||||
|
||||
class _RawTree:
|
||||
def __init__(self, replies: Mapping[str, tuple[int, bytes]]) -> None:
|
||||
self.replies = replies
|
||||
self.calls: list[str] = []
|
||||
self.headers: list[dict[str, str] | None] = []
|
||||
self.byte_limits: list[int | None] = []
|
||||
|
||||
async def get(
|
||||
self,
|
||||
url: str,
|
||||
*,
|
||||
headers: dict[str, str] | None = None,
|
||||
timeout: float | None = None,
|
||||
max_response_bytes: int | None = None,
|
||||
) -> httpx.Response:
|
||||
self.calls.append(url)
|
||||
self.headers.append(headers)
|
||||
self.byte_limits.append(max_response_bytes)
|
||||
status, body = self.replies.get(url, (404, b"404: Not Found"))
|
||||
return httpx.Response(status, content=body)
|
||||
|
||||
|
||||
class _Clock:
|
||||
def __init__(self) -> None:
|
||||
self.now = 1000.0
|
||||
|
||||
def __call__(self) -> float:
|
||||
return self.now
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _fetching_allowed(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("LITELLM_LOCAL_CODEX_CATALOG", raising=False)
|
||||
monkeypatch.delenv("LITELLM_OFFLINE_MODE", raising=False)
|
||||
monkeypatch.delenv("LITELLM_CODEX_CATALOG_BASE_URL", raising=False)
|
||||
monkeypatch.delenv("LITELLM_CODEX_CATALOG_OPENAI_URL", raising=False)
|
||||
|
||||
|
||||
def _source(
|
||||
tree: _RawTree,
|
||||
clock: _Clock | None = None,
|
||||
*,
|
||||
retry_seconds: float = 300.0,
|
||||
refresh_seconds: float = 3600.0,
|
||||
cached_versions: int = 16,
|
||||
) -> CodexCatalogSource:
|
||||
return CodexCatalogSource(
|
||||
client=tree,
|
||||
clock=clock if clock is not None else _Clock(),
|
||||
retry_seconds=retry_seconds,
|
||||
refresh_seconds=refresh_seconds,
|
||||
cached_versions=cached_versions,
|
||||
)
|
||||
|
||||
|
||||
def object_value(value: JsonValue) -> dict[str, JsonValue]:
|
||||
assert isinstance(value, dict), value
|
||||
return value
|
||||
|
||||
|
||||
def _entry(stock_models: Mapping[str, CodexStockModel], instructions: str, slug: str) -> dict[str, JsonValue]:
|
||||
body: Final = codex_models_response_json(
|
||||
(CodexCatalogRow(id=slug, mode="responses"),), stock=stock_models, instructions=instructions
|
||||
)
|
||||
entries: Final = JSON_OBJECT.validate_json(body.json)["models"]
|
||||
assert isinstance(entries, list)
|
||||
return object_value(entries[0])
|
||||
|
||||
|
||||
def test_the_requesting_release_catalog_serves_its_entry_for_a_model_the_vendored_pair_lacks() -> None:
|
||||
assert FRESH_SLUG not in vendored_codex_stock().models
|
||||
tree: Final = _RawTree(_tree(RELEASE))
|
||||
stock: Final = asyncio.run(_source(tree).stock(RELEASE))
|
||||
entry: Final = _entry(stock.models, stock.instructions, FRESH_SLUG)
|
||||
assert stock.source == f"github rust-v{RELEASE}"
|
||||
assert entry["shell_type"] == "shell_command" and entry["apply_patch_tool_type"] == "freeform", entry
|
||||
assert entry["supports_parallel_tool_calls"] is True, entry
|
||||
assert sorted(tree.calls) == sorted(_tree(RELEASE)), tree.calls
|
||||
|
||||
|
||||
def test_a_release_is_fetched_once_and_concurrent_requests_share_the_fetch() -> None:
|
||||
tree: Final = _RawTree(_tree(RELEASE))
|
||||
source: Final = _source(tree)
|
||||
|
||||
async def burst() -> list[str]:
|
||||
stocks: Final = await asyncio.gather(*(source.stock(RELEASE) for _ in range(5)))
|
||||
again: Final = await source.stock(RELEASE)
|
||||
return [stock.source for stock in (*stocks, again)]
|
||||
|
||||
assert asyncio.run(burst()) == [f"github rust-v{RELEASE}"] * 6
|
||||
assert len(tree.calls) == 2, tree.calls
|
||||
|
||||
|
||||
def test_a_release_without_a_tag_serves_the_vendored_pair_and_is_fetched_again_behind_it_after_the_window() -> None:
|
||||
tree: Final = _RawTree({})
|
||||
clock: Final = _Clock()
|
||||
source: Final = _source(tree, clock, retry_seconds=300)
|
||||
vendored: Final = vendored_codex_stock().source
|
||||
|
||||
async def scenario() -> tuple[str, ...]:
|
||||
first: Final = await source.stock(RELEASE)
|
||||
assert first.models == vendored_codex_stock().models
|
||||
assert _entry(first.models, first.instructions, FRESH_SLUG)["shell_type"] == "unified_exec"
|
||||
calls_after_first: Final = len(tree.calls)
|
||||
clock.now += 299
|
||||
inside_window: Final = await source.stock(RELEASE)
|
||||
assert len(tree.calls) == calls_after_first, "a failed fetch was retried inside its window"
|
||||
clock.now += 2
|
||||
tree.replies = _tree(RELEASE)
|
||||
past_window: Final = await source.stock(RELEASE)
|
||||
await asyncio.gather(*(asyncio.all_tasks() - {asyncio.current_task()}))
|
||||
refetched: Final = await source.stock(RELEASE)
|
||||
return first.source, inside_window.source, past_window.source, refetched.source
|
||||
|
||||
assert asyncio.run(asyncio.wait_for(scenario(), timeout=30)) == (
|
||||
vendored,
|
||||
vendored,
|
||||
vendored,
|
||||
f"github rust-v{RELEASE}",
|
||||
)
|
||||
|
||||
|
||||
def test_fetches_past_the_budget_inside_one_window_serve_the_vendored_pair_without_a_fetch() -> None:
|
||||
tree: Final = _RawTree({})
|
||||
clock: Final = _Clock()
|
||||
source: Final = _source(tree, clock, retry_seconds=300, cached_versions=2)
|
||||
|
||||
async def scenario() -> tuple[str, tuple[str, ...], tuple[str, ...]]:
|
||||
await source.stock("0.150.0")
|
||||
await source.stock("0.151.0")
|
||||
over_budget: Final = await source.stock("0.152.0")
|
||||
calls_inside_window: Final = tuple(tree.calls)
|
||||
clock.now += 301
|
||||
await source.stock("0.152.0")
|
||||
return over_budget.source, calls_inside_window, tuple(tree.calls)
|
||||
|
||||
over_budget, calls_inside_window, calls = asyncio.run(asyncio.wait_for(scenario(), timeout=30))
|
||||
assert over_budget == vendored_codex_stock().source
|
||||
assert not any("0.152.0" in url for url in calls_inside_window), calls_inside_window
|
||||
assert any("0.152.0" in url for url in calls), calls
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"replies",
|
||||
[
|
||||
pytest.param(_tree(RELEASE, catalog=b"<html>rate limited</html>"), id="catalog-not-json"),
|
||||
pytest.param(_tree(RELEASE, catalog=b'{"models": []}'), id="catalog-empty"),
|
||||
pytest.param(_tree(RELEASE, catalog=b'{"models": [{"slug": "x"}]}'), id="catalog-missing-fields"),
|
||||
pytest.param(_tree(RELEASE, prompt=b" "), id="prompt-blank"),
|
||||
pytest.param(
|
||||
{**_tree(RELEASE), **{k: (500, b"") for k in _tree(RELEASE) if k.endswith("prompt.md")}}, id="prompt-500"
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_a_release_whose_pair_does_not_parse_serves_the_vendored_pair(replies: Mapping[str, tuple[int, bytes]]) -> None:
|
||||
stock: Final = asyncio.run(_source(_RawTree(replies)).stock(RELEASE))
|
||||
assert stock.source == vendored_codex_stock().source
|
||||
assert FRESH_SLUG not in stock.models
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"error",
|
||||
[
|
||||
lambda url: httpx.ConnectError("offline", request=httpx.Request("GET", url)),
|
||||
lambda url: httpx.InvalidURL(f"Invalid non-printable ASCII character in URL {url!r}"),
|
||||
],
|
||||
)
|
||||
def test_a_transport_error_or_an_unusable_url_serves_the_vendored_pair(error: Callable[[str], Exception]) -> None:
|
||||
class _Down:
|
||||
async def get(
|
||||
self,
|
||||
url: str,
|
||||
*,
|
||||
headers: dict[str, str] | None = None,
|
||||
timeout: float | None = None,
|
||||
max_response_bytes: int | None = None,
|
||||
) -> httpx.Response:
|
||||
raise error(url)
|
||||
|
||||
stock: Final = asyncio.run(CodexCatalogSource(client=_Down(), clock=_Clock()).stock(RELEASE))
|
||||
assert stock.source == vendored_codex_stock().source
|
||||
|
||||
|
||||
@pytest.mark.parametrize("client_version", [None, "", "codex", "1.0", "0.166", "0.166.0 ", "0.166.0\n", "v0.166.0"])
|
||||
def test_a_value_that_is_no_release_version_serves_the_vendored_pair_without_a_fetch(
|
||||
client_version: str | None,
|
||||
) -> None:
|
||||
tree: Final = _RawTree(_tree(RELEASE))
|
||||
assert codex_release_tag(client_version) is None
|
||||
assert asyncio.run(_source(tree).stock(client_version)).source == vendored_codex_stock().source
|
||||
assert tree.calls == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("client_version", ["0.166.0", "0.167.0-alpha.4", "10.0.1"])
|
||||
def test_a_release_version_names_its_rust_tag(client_version: str) -> None:
|
||||
assert codex_release_tag(client_version) == f"rust-v{client_version}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("variable", ["LITELLM_LOCAL_CODEX_CATALOG", "LITELLM_OFFLINE_MODE"])
|
||||
def test_local_and_offline_settings_keep_the_vendored_pair_without_a_fetch(
|
||||
monkeypatch: pytest.MonkeyPatch, variable: str
|
||||
) -> None:
|
||||
monkeypatch.setenv(variable, "true")
|
||||
tree: Final = _RawTree(_tree(RELEASE))
|
||||
assert asyncio.run(_source(tree).stock(RELEASE)).source == vendored_codex_stock().source
|
||||
assert tree.calls == []
|
||||
|
||||
|
||||
def test_the_base_url_setting_points_the_fetch_at_a_mirror(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("LITELLM_CODEX_CATALOG_BASE_URL", "http://mirror.internal/codex/")
|
||||
mirrored: Final = {
|
||||
url.replace(CODEX_CATALOG_BASE_URL, "http://mirror.internal/codex"): reply
|
||||
for url, reply in _tree(RELEASE).items()
|
||||
}
|
||||
tree: Final = _RawTree(mirrored)
|
||||
assert asyncio.run(_source(tree).stock(RELEASE)).source == f"github rust-v{RELEASE}"
|
||||
assert sorted(tree.calls) == sorted(mirrored), tree.calls
|
||||
|
||||
|
||||
def test_the_cache_keeps_only_the_newest_releases() -> None:
|
||||
versions: Final = ("0.164.0", "0.165.0", "0.166.0")
|
||||
tree: Final = _RawTree(ChainMap(*map(_tree, versions)))
|
||||
clock: Final = _Clock()
|
||||
source: Final = _source(tree, clock, retry_seconds=300, cached_versions=2)
|
||||
|
||||
async def walk() -> None:
|
||||
for version in (*versions, versions[-1], versions[0]):
|
||||
clock.now += 301
|
||||
await source.stock(version)
|
||||
|
||||
asyncio.run(walk())
|
||||
oldest_models_url: Final = next(url for url in _tree(versions[0]) if url.endswith("models.json"))
|
||||
newest_models_url: Final = next(url for url in _tree(versions[-1]) if url.endswith("models.json"))
|
||||
assert tree.calls.count(oldest_models_url) == 2, tree.calls
|
||||
assert tree.calls.count(newest_models_url) == 1, tree.calls
|
||||
|
||||
|
||||
def test_releases_no_catalog_answered_leave_the_cache_before_a_fetched_one() -> None:
|
||||
tree: Final = _RawTree(_tree(RELEASE))
|
||||
clock: Final = _Clock()
|
||||
source: Final = _source(tree, clock, retry_seconds=300, cached_versions=2)
|
||||
|
||||
async def walk() -> None:
|
||||
await source.stock(RELEASE)
|
||||
for minor in range(201, 205):
|
||||
clock.now += 301
|
||||
await source.stock(f"0.{minor}.0")
|
||||
clock.now += 301
|
||||
await source.stock(RELEASE)
|
||||
|
||||
asyncio.run(walk())
|
||||
models_url: Final = next(url for url in _tree(RELEASE) if url.endswith("models.json"))
|
||||
assert tree.calls.count(models_url) == 1, tree.calls
|
||||
|
||||
|
||||
def test_a_release_whose_fetch_just_failed_keeps_its_retry_time_when_the_cache_is_full_of_fetched_ones() -> None:
|
||||
older: Final = "0.165.0"
|
||||
tree: Final = _RawTree({**_tree(older), **_tree(RELEASE)})
|
||||
clock: Final = _Clock()
|
||||
source: Final = _source(tree, clock, retry_seconds=300, cached_versions=2)
|
||||
missing: Final = "0.204.0"
|
||||
|
||||
async def walk() -> None:
|
||||
await source.stock(older)
|
||||
clock.now += 301
|
||||
await source.stock(RELEASE)
|
||||
clock.now += 301
|
||||
await source.stock(missing)
|
||||
clock.now += 100
|
||||
await source.stock(missing)
|
||||
|
||||
asyncio.run(walk())
|
||||
missing_url: Final = next(url for url in tree.calls if f"rust-v{missing}/" in url and url.endswith("models.json"))
|
||||
assert tree.calls.count(missing_url) == 1, tree.calls
|
||||
|
||||
|
||||
def test_a_catalog_url_that_redirects_to_itself_serves_the_vendored_pair_and_waits_out_the_retry_window() -> None:
|
||||
hops: Final[list[str]] = []
|
||||
|
||||
def redirect_to_itself(request: httpx.Request) -> httpx.Response:
|
||||
hops.append(str(request.url))
|
||||
return httpx.Response(302, headers={"location": str(request.url)})
|
||||
|
||||
source: Final = CodexCatalogSource(
|
||||
client=AsyncHTTPHandler(transport=httpx.MockTransport(redirect_to_itself)), clock=_Clock()
|
||||
)
|
||||
|
||||
async def ask_twice() -> tuple[str, int, str]:
|
||||
first: Final = await source.stock(RELEASE)
|
||||
hops_for_first: Final = len(hops)
|
||||
second: Final = await source.stock(RELEASE)
|
||||
return first.source, hops_for_first, second.source
|
||||
|
||||
first_source, hops_for_first, second_source = asyncio.run(ask_twice())
|
||||
assert first_source == second_source == vendored_codex_stock().source
|
||||
assert hops_for_first > 0
|
||||
assert len(hops) == hops_for_first
|
||||
|
||||
|
||||
OPENAI_KEY: Final = "sk-unit-openai-key"
|
||||
OPENAI_ENTRY: Final = {**FRESH_ENTRY, "shell_type": "unified_exec", "base_instructions": "# OpenAI live prompt"}
|
||||
OPENAI_CATALOG: Final = json.dumps({"models": [OPENAI_ENTRY]}).encode()
|
||||
|
||||
|
||||
def _openai_url(version: str) -> str:
|
||||
return f"{CODEX_CATALOG_OPENAI_URL}?client_version={version}"
|
||||
|
||||
|
||||
def test_an_openai_key_fetches_the_live_catalog_from_openai_with_that_key_and_skips_github() -> None:
|
||||
tree: Final = _RawTree({**_tree(RELEASE), _openai_url(RELEASE): (200, OPENAI_CATALOG)})
|
||||
stock: Final = asyncio.run(_source(tree).stock(RELEASE, openai_api_key=OPENAI_KEY))
|
||||
entry: Final = _entry(stock.models, stock.instructions, FRESH_SLUG)
|
||||
assert stock.source == f"openai {RELEASE}" and stock.instructions == codex_base_instructions(), stock.source
|
||||
assert entry["shell_type"] == "unified_exec" and entry["base_instructions"] == "# OpenAI live prompt", entry
|
||||
assert tree.calls == [_openai_url(RELEASE)] and tree.headers == [{"Authorization": f"Bearer {OPENAI_KEY}"}]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("status", [401, 500])
|
||||
def test_a_refused_openai_fetch_falls_back_to_the_release_tag_on_github(status: int) -> None:
|
||||
tree: Final = _RawTree({**_tree(RELEASE), _openai_url(RELEASE): (status, b"{}")})
|
||||
stock: Final = asyncio.run(_source(tree).stock(RELEASE, openai_api_key=OPENAI_KEY))
|
||||
assert stock.source == f"github rust-v{RELEASE}" and FRESH_SLUG in stock.models
|
||||
assert tree.calls[0] == _openai_url(RELEASE) and sorted(tree.calls[1:]) == sorted(_tree(RELEASE)), tree.calls
|
||||
|
||||
|
||||
def test_an_openai_answer_that_never_finishes_falls_back_to_the_release_tag_within_the_fetch_timeout() -> None:
|
||||
class _NeverFinishingOpenAI(_RawTree):
|
||||
async def get(
|
||||
self,
|
||||
url: str,
|
||||
*,
|
||||
headers: dict[str, str] | None = None,
|
||||
timeout: float | None = None,
|
||||
max_response_bytes: int | None = None,
|
||||
) -> httpx.Response:
|
||||
if url == _openai_url(RELEASE):
|
||||
self.calls.append(url)
|
||||
await asyncio.Event().wait()
|
||||
return await super().get(url, headers=headers, timeout=timeout, max_response_bytes=max_response_bytes)
|
||||
|
||||
tree: Final = _NeverFinishingOpenAI(_tree(RELEASE))
|
||||
source: Final = CodexCatalogSource(client=tree, clock=_Clock(), timeout=0.05, response_deadline=30)
|
||||
stock: Final = asyncio.run(asyncio.wait_for(source.stock(RELEASE, openai_api_key=OPENAI_KEY), timeout=5))
|
||||
assert stock.source == f"github rust-v{RELEASE}" and FRESH_SLUG in stock.models
|
||||
|
||||
|
||||
def test_an_empty_openai_url_setting_keeps_the_key_home_and_fetches_from_github(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setenv("LITELLM_CODEX_CATALOG_OPENAI_URL", "")
|
||||
tree: Final = _RawTree({**_tree(RELEASE), _openai_url(RELEASE): (200, OPENAI_CATALOG)})
|
||||
stock: Final = asyncio.run(_source(tree).stock(RELEASE, openai_api_key=OPENAI_KEY))
|
||||
assert stock.source == f"github rust-v{RELEASE}"
|
||||
assert sorted(tree.calls) == sorted(_tree(RELEASE)) and tree.headers == [None, None], tree.calls
|
||||
|
||||
|
||||
def test_without_an_openai_key_openai_is_never_asked() -> None:
|
||||
tree: Final = _RawTree({**_tree(RELEASE), _openai_url(RELEASE): (200, OPENAI_CATALOG)})
|
||||
stock: Final = asyncio.run(_source(tree).stock(RELEASE))
|
||||
assert stock.source == f"github rust-v{RELEASE}"
|
||||
assert _openai_url(RELEASE) not in tree.calls and tree.headers == [None, None], tree.calls
|
||||
|
||||
|
||||
def test_every_fetch_caps_the_download_at_the_catalog_byte_limit_while_it_streams() -> None:
|
||||
tree: Final = _RawTree(_tree(RELEASE))
|
||||
asyncio.run(_source(tree).stock(RELEASE))
|
||||
assert tree.byte_limits == [CODEX_CATALOG_MAX_BYTES] * len(tree.calls) and tree.calls, tree.byte_limits
|
||||
|
||||
|
||||
def test_a_release_asked_for_while_the_in_flight_fetches_are_at_the_cap_gets_the_vendored_pair_without_a_fetch() -> (
|
||||
None
|
||||
):
|
||||
class _Hanging(_RawTree):
|
||||
async def get(
|
||||
self,
|
||||
url: str,
|
||||
*,
|
||||
headers: dict[str, str] | None = None,
|
||||
timeout: float | None = None,
|
||||
max_response_bytes: int | None = None,
|
||||
) -> httpx.Response:
|
||||
self.calls.append(url)
|
||||
await asyncio.Event().wait()
|
||||
raise AssertionError("unreachable")
|
||||
|
||||
async def scenario() -> tuple[str, list[str]]:
|
||||
tree: Final = _Hanging({})
|
||||
source: Final = _source(tree, cached_versions=2)
|
||||
hung: Final = [asyncio.ensure_future(source.stock(f"0.{n}.0")) for n in (150, 151)]
|
||||
await asyncio.sleep(0)
|
||||
over_cap: Final = await source.stock("0.152.0")
|
||||
for task in hung:
|
||||
task.cancel()
|
||||
return over_cap.source, list(tree.calls)
|
||||
|
||||
source_label, calls = asyncio.run(scenario())
|
||||
assert source_label == vendored_codex_stock().source and not any("0.152.0" in url for url in calls), calls
|
||||
|
||||
|
||||
def test_a_fetch_that_outlasts_the_response_deadline_serves_the_vendored_pair_then_the_release_once_it_lands() -> None:
|
||||
class _Gated(_RawTree):
|
||||
def __init__(self, replies: Mapping[str, tuple[int, bytes]]) -> None:
|
||||
super().__init__(replies)
|
||||
self.gate = asyncio.Event()
|
||||
|
||||
async def get(
|
||||
self,
|
||||
url: str,
|
||||
*,
|
||||
headers: dict[str, str] | None = None,
|
||||
timeout: float | None = None,
|
||||
max_response_bytes: int | None = None,
|
||||
) -> httpx.Response:
|
||||
self.calls.append(url)
|
||||
await self.gate.wait()
|
||||
status, body = self.replies.get(url, (404, b"404: Not Found"))
|
||||
return httpx.Response(status, content=body)
|
||||
|
||||
async def scenario() -> tuple[str, str, int]:
|
||||
tree: Final = _Gated(_tree(RELEASE))
|
||||
source: Final = CodexCatalogSource(client=tree, clock=_Clock(), response_deadline=0.01)
|
||||
early: Final = await source.stock(RELEASE)
|
||||
tree.gate.set()
|
||||
await asyncio.gather(*(asyncio.all_tasks() - {asyncio.current_task()}))
|
||||
late: Final = await source.stock(RELEASE)
|
||||
return early.source, late.source, len(tree.calls)
|
||||
|
||||
assert asyncio.run(asyncio.wait_for(scenario(), timeout=30)) == (
|
||||
vendored_codex_stock().source,
|
||||
f"github rust-v{RELEASE}",
|
||||
2,
|
||||
)
|
||||
|
||||
|
||||
def test_a_download_the_client_cuts_at_the_byte_limit_falls_through_to_the_next_source() -> None:
|
||||
class _CuttingTree(_RawTree):
|
||||
async def get(
|
||||
self,
|
||||
url: str,
|
||||
*,
|
||||
headers: dict[str, str] | None = None,
|
||||
timeout: float | None = None,
|
||||
max_response_bytes: int | None = None,
|
||||
) -> httpx.Response:
|
||||
if url == _openai_url(RELEASE):
|
||||
self.calls.append(url)
|
||||
raise HTTPResponseLimitError(f"{url} exceeded {max_response_bytes} bytes")
|
||||
return await super().get(url, headers=headers, timeout=timeout, max_response_bytes=max_response_bytes)
|
||||
|
||||
tree: Final = _CuttingTree(_tree(RELEASE))
|
||||
stock: Final = asyncio.run(_source(tree).stock(RELEASE, openai_api_key=OPENAI_KEY))
|
||||
assert stock.source == f"github rust-v{RELEASE}" and tree.calls[0] == _openai_url(RELEASE), tree.calls
|
||||
|
||||
|
||||
def test_catalogs_fetched_with_different_openai_keys_are_cached_apart() -> None:
|
||||
tree: Final = _RawTree({**_tree(RELEASE), _openai_url(RELEASE): (200, OPENAI_CATALOG)})
|
||||
|
||||
async def scenario() -> None:
|
||||
source: Final = _source(tree)
|
||||
await source.stock(RELEASE, openai_api_key=OPENAI_KEY)
|
||||
await source.stock(RELEASE, openai_api_key=OPENAI_KEY)
|
||||
await source.stock(RELEASE, openai_api_key="sk-unit-other-key")
|
||||
|
||||
asyncio.run(scenario())
|
||||
assert tree.headers == [
|
||||
{"Authorization": f"Bearer {OPENAI_KEY}"},
|
||||
{"Authorization": "Bearer sk-unit-other-key"},
|
||||
], tree.headers
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("openai_api_key", "source"), [(None, f"github rust-v{RELEASE}"), (OPENAI_KEY, f"openai {RELEASE}")]
|
||||
)
|
||||
def test_a_fetched_catalog_wins_for_the_models_it_lists_and_the_vendored_entry_stays_for_the_rest(
|
||||
openai_api_key: str | None, source: str
|
||||
) -> None:
|
||||
vendored: Final = vendored_codex_stock().models
|
||||
listed, missing = tuple(vendored)[:2]
|
||||
catalog: Final = json.dumps({"models": [{**FRESH_ENTRY, "slug": listed, "display_name": "Release"}]}).encode()
|
||||
tree: Final = _RawTree({**_tree(RELEASE, catalog=catalog), _openai_url(RELEASE): (200, catalog)})
|
||||
stock: Final = asyncio.run(_source(tree).stock(RELEASE, openai_api_key=openai_api_key))
|
||||
assert stock.source == source
|
||||
assert stock.models[listed].display_name == "Release" != vendored[listed].display_name
|
||||
assert stock.models[missing] == vendored[missing]
|
||||
|
||||
|
||||
def _settled() -> Awaitable[list[object]]:
|
||||
return asyncio.gather(*(asyncio.all_tasks() - {asyncio.current_task()}))
|
||||
|
||||
|
||||
def _openai_catalog(display_name: str) -> bytes:
|
||||
return json.dumps({"models": [{**OPENAI_ENTRY, "display_name": display_name}]}).encode()
|
||||
|
||||
|
||||
def test_openai_catalog_is_refetched_after_the_refresh_window_and_a_failed_refresh_keeps_it_over_the_release_tag() -> (
|
||||
None
|
||||
):
|
||||
tree: Final = _RawTree({_openai_url(RELEASE): (200, _openai_catalog("First"))})
|
||||
clock: Final = _Clock()
|
||||
source: Final = _source(tree, clock, retry_seconds=300, refresh_seconds=3600)
|
||||
|
||||
async def names() -> list[str]:
|
||||
await _settled()
|
||||
return [(await source.stock(RELEASE, openai_api_key=OPENAI_KEY)).models[FRESH_SLUG].display_name]
|
||||
|
||||
async def scenario() -> tuple[list[str], ...]:
|
||||
first: Final = await names()
|
||||
clock.now += 3599
|
||||
tree.replies = {_openai_url(RELEASE): (200, _openai_catalog("Second"))}
|
||||
inside_window: Final = await names()
|
||||
clock.now += 2
|
||||
await source.stock(RELEASE, openai_api_key=OPENAI_KEY)
|
||||
refreshed: Final = await names()
|
||||
clock.now += 3601
|
||||
tree.replies = _tree(RELEASE)
|
||||
await source.stock(RELEASE, openai_api_key=OPENAI_KEY)
|
||||
refresh_failed: Final = await names()
|
||||
clock.now += 301
|
||||
await source.stock(RELEASE, openai_api_key=OPENAI_KEY)
|
||||
retry_failed: Final = await names()
|
||||
clock.now += 301
|
||||
tree.replies = {_openai_url(RELEASE): (200, _openai_catalog("Third"))}
|
||||
await source.stock(RELEASE, openai_api_key=OPENAI_KEY)
|
||||
retried: Final = await names()
|
||||
return first, inside_window, refreshed, refresh_failed, retry_failed, retried
|
||||
|
||||
assert asyncio.run(asyncio.wait_for(scenario(), timeout=30)) == (
|
||||
["First"],
|
||||
["First"],
|
||||
["Second"],
|
||||
["Second"],
|
||||
["Second"],
|
||||
["Third"],
|
||||
)
|
||||
|
||||
|
||||
def test_a_github_answer_after_openai_failed_asks_openai_again_after_the_retry_window() -> None:
|
||||
tree: Final = _RawTree({**_tree(RELEASE), _openai_url(RELEASE): (500, b"{}")})
|
||||
clock: Final = _Clock()
|
||||
source: Final = _source(tree, clock, retry_seconds=300, refresh_seconds=3600)
|
||||
|
||||
async def scenario() -> tuple[str, str, str]:
|
||||
first: Final = await source.stock(RELEASE, openai_api_key=OPENAI_KEY)
|
||||
clock.now += 299
|
||||
inside_window: Final = await source.stock(RELEASE, openai_api_key=OPENAI_KEY)
|
||||
clock.now += 2
|
||||
tree.replies = {_openai_url(RELEASE): (200, OPENAI_CATALOG)}
|
||||
await source.stock(RELEASE, openai_api_key=OPENAI_KEY)
|
||||
await _settled()
|
||||
recovered: Final = await source.stock(RELEASE, openai_api_key=OPENAI_KEY)
|
||||
return first.source, inside_window.source, recovered.source
|
||||
|
||||
assert asyncio.run(asyncio.wait_for(scenario(), timeout=30)) == (
|
||||
f"github rust-v{RELEASE}",
|
||||
f"github rust-v{RELEASE}",
|
||||
f"openai {RELEASE}",
|
||||
)
|
||||
assert tree.calls.count(_openai_url(RELEASE)) == 2, tree.calls
|
||||
|
||||
|
||||
def test_a_release_tag_fetched_without_an_openai_key_is_never_fetched_again() -> None:
|
||||
tree: Final = _RawTree(_tree(RELEASE))
|
||||
clock: Final = _Clock()
|
||||
source: Final = _source(tree, clock, retry_seconds=300, refresh_seconds=3600)
|
||||
|
||||
async def scenario() -> str:
|
||||
await source.stock(RELEASE)
|
||||
clock.now += 10 * 3600
|
||||
await source.stock(RELEASE)
|
||||
await _settled()
|
||||
return (await source.stock(RELEASE)).source
|
||||
|
||||
assert asyncio.run(asyncio.wait_for(scenario(), timeout=30)) == f"github rust-v{RELEASE}"
|
||||
assert len(tree.calls) == 2, tree.calls
|
||||
|
||||
|
||||
def test_openai_entries_drop_the_prompt_copy_of_their_template_so_a_codex_capped_at_1_mib_lists_every_model() -> None:
|
||||
prompt: Final = "p" * 21_000
|
||||
slugs: Final = tuple(f"gpt-big-{index}" for index in range(34))
|
||||
catalog: Final = json.dumps(
|
||||
{
|
||||
"models": [
|
||||
{
|
||||
**OPENAI_ENTRY,
|
||||
"slug": slug,
|
||||
"base_instructions": prompt,
|
||||
"model_messages": {"instructions_template": prompt},
|
||||
}
|
||||
for slug in slugs
|
||||
]
|
||||
}
|
||||
).encode()
|
||||
tree: Final = _RawTree({_openai_url("0.165.0"): (200, catalog)})
|
||||
stock: Final = asyncio.run(_source(tree).stock("0.165.0", openai_api_key=OPENAI_KEY))
|
||||
body: Final = codex_models_response_json(
|
||||
tuple(CodexCatalogRow(id=slug, mode="responses") for slug in slugs),
|
||||
stock=stock.models,
|
||||
instructions=stock.instructions,
|
||||
client_version="0.165.0",
|
||||
)
|
||||
entries: Final = JSON_OBJECT.validate_json(body.json)["models"]
|
||||
assert isinstance(entries, list)
|
||||
listed: Final = tuple(map(object_value, entries))
|
||||
assert tuple(entry["slug"] for entry in listed) == slugs
|
||||
assert all(entry["base_instructions"] == "" for entry in listed), listed[0]["base_instructions"]
|
||||
assert all(object_value(entry["model_messages"])["instructions_template"] == prompt for entry in listed)
|
||||
|
|
@ -14,11 +14,14 @@ from litellm.proxy.common_utils.codex_model_catalog import (
|
|||
CodexStockModel,
|
||||
CodexStockUpgrade,
|
||||
bundled_codex_models,
|
||||
codex_base_instructions,
|
||||
codex_catalog_openai_key,
|
||||
codex_catalog_rows,
|
||||
codex_model_list_body,
|
||||
codex_models_response_json,
|
||||
configured_service_tiers,
|
||||
)
|
||||
from litellm.types.utils import CredentialItem
|
||||
|
||||
_PROMPT = "base prompt"
|
||||
_FAST = {"id": "priority", "name": "Fast", "description": "2x speed, increased usage"}
|
||||
|
|
@ -672,6 +675,27 @@ def test_unlimited_body_keeps_every_entry():
|
|||
assert CODEX_CATALOG_BYTE_LIMIT == 1024 * 1024
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("client_version", "capped"),
|
||||
[(None, True), ("0.165.0", True), ("0.166.0", False), ("0.167.0-alpha.4", False), ("not-a-version", True)],
|
||||
)
|
||||
def test_the_catalog_is_cut_at_the_byte_limit_only_for_a_codex_that_enforces_one(
|
||||
client_version: str | None, capped: bool
|
||||
) -> None:
|
||||
names: Final = tuple(
|
||||
f"chat-model-{index}" for index in range(CODEX_CATALOG_BYTE_LIMIT // len(codex_base_instructions()) + 2)
|
||||
)
|
||||
listing: Final = tuple(
|
||||
{"id": name, "object": "model", "created": 0, "owned_by": "x", "mode": "chat"} for name in names
|
||||
)
|
||||
body: Final = codex_model_list_body(
|
||||
listing, tuple((name, name) for name in names), None, client_version=client_version
|
||||
)
|
||||
slugs: Final = [entry["slug"] for entry in json.loads(body)["models"]]
|
||||
assert (len(body.encode()) <= CODEX_CATALOG_BYTE_LIMIT) is capped, len(body.encode())
|
||||
assert (slugs == list(names)) is not capped, slugs
|
||||
|
||||
|
||||
def test_vendored_codex_catalog_entries_decode_on_codex():
|
||||
"""Every stock entry served keeps the `model_messages` Codex's `ModelsResponse` decoder
|
||||
needs in place of `base_instructions`, and the catalog is read from the vendored file."""
|
||||
|
|
@ -944,3 +968,231 @@ def test_codex_tool_search_is_offered_off_the_route_and_model_info_and_only_when
|
|||
"mixed": False,
|
||||
}
|
||||
assert models[0]["context_window"] == 1000000
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def catalog_fetching(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("LITELLM_LOCAL_CODEX_CATALOG", raising=False)
|
||||
monkeypatch.delenv("LITELLM_OFFLINE_MODE", raising=False)
|
||||
|
||||
|
||||
def _router_with(*litellm_params: Mapping[str, str | bool]) -> litellm.Router:
|
||||
return litellm.Router(
|
||||
model_list=[
|
||||
{"model_name": f"catalog-model-{index}", "litellm_params": dict(params)}
|
||||
for index, params in enumerate(litellm_params)
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("catalog_fetching")
|
||||
def test_the_catalog_key_is_the_listed_openai_deployments_own_and_the_environments_only_for_a_keyless_one(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "sk-from-env")
|
||||
monkeypatch.setenv("CATALOG_OPENAI_KEY", "sk-from-secret-ref")
|
||||
router: Final = _router_with(
|
||||
{"model": "anthropic/claude-sonnet-4-5", "api_key": "sk-ant-not-openai"},
|
||||
{"model": "openai/gpt-5.5", "api_key": "os.environ/CATALOG_OPENAI_KEY"},
|
||||
{"model": "openai/gpt-5.5", "api_key": "sk-second-openai"},
|
||||
{"model": "openai/gpt-5.5"},
|
||||
)
|
||||
entries: Final = [(f"catalog-model-{index}", f"catalog-model-{index}") for index in range(4)]
|
||||
assert codex_catalog_openai_key(router, entries) == "sk-from-secret-ref"
|
||||
assert codex_catalog_openai_key(router, entries[:1]) is None
|
||||
assert codex_catalog_openai_key(router, entries[3:]) == "sk-from-env"
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("catalog_fetching")
|
||||
def test_a_bare_openai_model_name_supplies_its_key_and_a_bare_alias_of_another_provider_does_not(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
|
||||
router: Final = _router_with(
|
||||
{"model": "claude-sonnet-4-5", "api_key": "sk-ant-bare-alias"},
|
||||
{"model": "gpt-5.5", "api_key": "sk-bare-openai"},
|
||||
{"model": "gpt-5.5", "custom_llm_provider": "azure", "api_key": "sk-azure-bare"},
|
||||
)
|
||||
entries: Final = [(f"catalog-model-{index}", f"catalog-model-{index}") for index in range(3)]
|
||||
assert codex_catalog_openai_key(router, entries) == "sk-bare-openai"
|
||||
assert codex_catalog_openai_key(router, entries[:1]) is None
|
||||
assert codex_catalog_openai_key(router, entries[2:]) is None
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("catalog_fetching")
|
||||
def test_the_catalog_key_is_absent_without_an_openai_deployment_or_environment_key(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
|
||||
router: Final = _router_with({"model": "anthropic/claude-sonnet-4-5", "api_key": "sk-ant-not-openai"})
|
||||
assert codex_catalog_openai_key(router, [("catalog-model-0", "catalog-model-0")]) is None
|
||||
assert codex_catalog_openai_key(None, [("catalog-model-0", "catalog-model-0")]) is None
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("catalog_fetching")
|
||||
def test_a_key_for_an_openai_compatible_server_or_a_redirected_environment_never_goes_to_openai(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "sk-from-env")
|
||||
monkeypatch.setenv("OPENAI_BASE_URL", "https://llm-gateway.internal.example/v1")
|
||||
router: Final = _router_with(
|
||||
{
|
||||
"model": "openai/local-llama",
|
||||
"api_key": "sk-internal-server",
|
||||
"api_base": "https://inference.internal.example/v1",
|
||||
},
|
||||
{"model": "openai/gpt-5.5", "api_key": "sk-through-the-gateway"},
|
||||
{"model": "openai/gpt-5.5", "api_key": "sk-real-openai", "api_base": "https://api.openai.com/v1"},
|
||||
{"model": "openai/gpt-5.5", "api_key": "sk-eu-openai", "api_base": "https://eu.api.openai.com/v1"},
|
||||
)
|
||||
entries: Final = [(f"catalog-model-{index}", f"catalog-model-{index}") for index in range(4)]
|
||||
assert codex_catalog_openai_key(router, entries[:2]) is None
|
||||
assert codex_catalog_openai_key(router, entries) == "sk-real-openai"
|
||||
assert codex_catalog_openai_key(router, entries[3:]) is None
|
||||
monkeypatch.delenv("OPENAI_BASE_URL")
|
||||
monkeypatch.setattr(litellm, "api_base", "https://llm-gateway.internal.example/v1")
|
||||
assert codex_catalog_openai_key(router, entries[:2]) is None
|
||||
monkeypatch.setattr(litellm, "api_base", None)
|
||||
assert codex_catalog_openai_key(router, entries[:2]) == "sk-through-the-gateway"
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("catalog_fetching")
|
||||
def test_a_fedramp_key_or_a_gov_openai_base_never_goes_to_openais_commercial_catalog(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "sk-proj-fedramp-from-env")
|
||||
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
|
||||
monkeypatch.delenv("OPENAI_API_BASE", raising=False)
|
||||
router: Final = _router_with(
|
||||
{"model": "openai/gpt-5.5", "api_key": "sk-gov-base", "api_base": "https://gov.api.openai.com/v1"},
|
||||
{"model": "openai/gpt-5.5", "api_key": "sk-proj-fedramp-own"},
|
||||
{"model": "openai/gpt-5.5", "api_key": "sk-svcacct-fedramp-own"},
|
||||
{"model": "openai/gpt-5.5"},
|
||||
{"model": "openai/gpt-5.5", "api_key": "sk-commercial"},
|
||||
)
|
||||
entries: Final = [(f"catalog-model-{index}", f"catalog-model-{index}") for index in range(5)]
|
||||
assert codex_catalog_openai_key(router, entries[:4]) is None
|
||||
assert codex_catalog_openai_key(router, entries) == "sk-commercial"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("setting", ["LITELLM_LOCAL_CODEX_CATALOG", "LITELLM_OFFLINE_MODE"])
|
||||
@pytest.mark.usefixtures("catalog_fetching")
|
||||
def test_no_catalog_key_is_looked_up_while_the_catalog_stays_local(
|
||||
monkeypatch: pytest.MonkeyPatch, setting: str
|
||||
) -> None:
|
||||
monkeypatch.setenv("CATALOG_OPENAI_KEY", "sk-from-secret-ref")
|
||||
router: Final = _router_with({"model": "openai/gpt-5.5", "api_key": "os.environ/CATALOG_OPENAI_KEY"})
|
||||
entries: Final = [("catalog-model-0", "catalog-model-0")]
|
||||
assert codex_catalog_openai_key(router, entries) == "sk-from-secret-ref"
|
||||
monkeypatch.setenv(setting, "True")
|
||||
assert codex_catalog_openai_key(router, entries) is None
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("catalog_fetching")
|
||||
def test_a_base_url_or_a_credential_naming_another_server_keeps_the_key_from_openai(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
|
||||
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
|
||||
monkeypatch.delenv("OPENAI_API_BASE", raising=False)
|
||||
monkeypatch.setattr(litellm, "api_base", None)
|
||||
monkeypatch.setattr(
|
||||
litellm,
|
||||
"credential_list",
|
||||
[
|
||||
CredentialItem(
|
||||
credential_name="internal-server",
|
||||
credential_values={
|
||||
"api_key": "sk-internal-credential",
|
||||
"api_base": "https://inference.internal.example/v1",
|
||||
},
|
||||
credential_info={},
|
||||
),
|
||||
CredentialItem(
|
||||
credential_name="openai-org", credential_values={"api_key": "sk-openai-credential"}, credential_info={}
|
||||
),
|
||||
],
|
||||
)
|
||||
router: Final = _router_with(
|
||||
{
|
||||
"model": "openai/gpt-5.5",
|
||||
"api_key": "sk-base-url-server",
|
||||
"base_url": "https://inference.internal.example/v1",
|
||||
},
|
||||
{"model": "openai/gpt-5.5", "litellm_credential_name": "internal-server"},
|
||||
{"model": "openai/gpt-5.5", "api_key": "sk-own-key", "litellm_credential_name": "internal-server"},
|
||||
{"model": "openai/gpt-5.5", "litellm_credential_name": "openai-org"},
|
||||
)
|
||||
entries: Final = [(f"catalog-model-{index}", f"catalog-model-{index}") for index in range(4)]
|
||||
assert codex_catalog_openai_key(router, entries[:3]) is None
|
||||
assert codex_catalog_openai_key(router, entries) == "sk-openai-credential"
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("catalog_fetching")
|
||||
def test_a_deployments_own_key_wins_over_its_named_credentials_key_as_on_the_call_path(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
|
||||
monkeypatch.delenv("OPENAI_API_BASE", raising=False)
|
||||
monkeypatch.setattr(litellm, "api_base", None)
|
||||
monkeypatch.setattr(
|
||||
litellm,
|
||||
"credential_list",
|
||||
[
|
||||
CredentialItem(
|
||||
credential_name="openai-org", credential_values={"api_key": "sk-openai-credential"}, credential_info={}
|
||||
)
|
||||
],
|
||||
)
|
||||
router: Final = _router_with(
|
||||
{"model": "openai/gpt-5.5", "api_key": "sk-own-key", "litellm_credential_name": "openai-org"}
|
||||
)
|
||||
assert codex_catalog_openai_key(router, [("catalog-model-0", "catalog-model-0")]) == "sk-own-key"
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("catalog_fetching")
|
||||
def test_an_api_base_that_does_not_parse_counts_as_another_server(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
|
||||
monkeypatch.delenv("OPENAI_API_BASE", raising=False)
|
||||
monkeypatch.setattr(litellm, "api_base", None)
|
||||
router: Final = _router_with(
|
||||
{"model": "openai/gpt-5.5", "api_key": "sk-unparseable-base", "api_base": "https://[my-vllm]:8000/v1"},
|
||||
{"model": "openai/gpt-5.5", "api_key": "sk-plain-openai"},
|
||||
)
|
||||
entries: Final = [(f"catalog-model-{index}", f"catalog-model-{index}") for index in range(2)]
|
||||
assert codex_catalog_openai_key(router, entries[:1]) is None
|
||||
assert codex_catalog_openai_key(router, entries) == "sk-plain-openai"
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("catalog_fetching")
|
||||
def test_an_oidc_key_reference_is_never_sent_as_the_catalog_key(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
|
||||
monkeypatch.delenv("OPENAI_API_BASE", raising=False)
|
||||
monkeypatch.setattr(litellm, "api_base", None)
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "sk-from-env")
|
||||
monkeypatch.setenv("CATALOG_OIDC_KEY", "sk-behind-the-reference")
|
||||
router: Final = _router_with(
|
||||
{"model": "openai/gpt-5.5", "api_key": "oidc/env/CATALOG_OIDC_KEY"},
|
||||
{"model": "openai/gpt-5.5", "api_key": "sk-plain-openai"},
|
||||
)
|
||||
entries: Final = [(f"catalog-model-{index}", f"catalog-model-{index}") for index in range(2)]
|
||||
assert codex_catalog_openai_key(router, entries[:1]) is None
|
||||
assert codex_catalog_openai_key(router, entries) == "sk-plain-openai"
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("catalog_fetching")
|
||||
def test_a_deployment_sent_through_a_litellm_proxy_keeps_its_key_from_openai(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
|
||||
monkeypatch.delenv("OPENAI_API_BASE", raising=False)
|
||||
monkeypatch.delenv("USE_LITELLM_PROXY", raising=False)
|
||||
monkeypatch.setattr(litellm, "api_base", None)
|
||||
monkeypatch.setattr(litellm, "use_litellm_proxy", False)
|
||||
router: Final = _router_with(
|
||||
{"model": "openai/gpt-5.5", "api_key": "sk-downstream-proxy", "use_litellm_proxy": True},
|
||||
{"model": "openai/gpt-5.5", "api_key": "sk-plain-openai"},
|
||||
)
|
||||
entries: Final = [(f"catalog-model-{index}", f"catalog-model-{index}") for index in range(2)]
|
||||
assert codex_catalog_openai_key(router, entries[:1]) is None
|
||||
assert codex_catalog_openai_key(router, entries) == "sk-plain-openai"
|
||||
monkeypatch.setenv("USE_LITELLM_PROXY", "True")
|
||||
assert codex_catalog_openai_key(router, entries) is None
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue