From 2fbadede4ce84dfb8245db40b0368e0966be3a3c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 14:52:36 -0700 Subject: [PATCH] feat(lite): offer each model group's reasoning efforts in pi, OpenCode and Codex lite pi, lite opencode and lite codex now read supported_reasoning_efforts from /model_group/info and write them into each client's model config. pi gets reasoning and a thinkingLevelMap, OpenCode gets reasoning_options plus one variant per effort, and Codex fallback entries get supported_reasoning_levels with a default level. The proxy-served Codex catalog on /v1/models?client_version= offers the same levels Codex before 0.138.0 fails the whole catalog on an effort it cannot name, so those releases, and an unreadable version, get only the six efforts every release parses. A group with unknown efforts keeps today's output, and a group with none is written as non-reasoning The strength order and nearest_declared_reasoning_effort move to an SDK-free module so the lite CLI, which loads without initializing the SDK, can import the Codex catalog --- litellm/proxy/client/cli/commands/agents.py | 124 ++++++++++++-- litellm/proxy/client/cli/commands/pi.py | 32 +++- .../proxy/common_utils/codex_model_catalog.py | 90 ++++++++-- litellm/proxy/proxy_server.py | 21 ++- .../reasoning_effort_capability.py | 41 +---- .../router_utils/reasoning_effort_strength.py | 45 +++++ tests/unit/proxy/client/cli/test_agents.py | 156 +++++++++++++++--- tests/unit/proxy/client/cli/test_pi.py | 69 +++++++- .../common_utils/test_codex_model_catalog.py | 131 ++++++++++++++- .../proxy/proxy_server/test_routes_models.py | 25 ++- .../test_reasoning_effort_capability.py | 2 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 12 +- 12 files changed, 646 insertions(+), 102 deletions(-) create mode 100644 litellm/router_utils/reasoning_effort_strength.py diff --git a/litellm/proxy/client/cli/commands/agents.py b/litellm/proxy/client/cli/commands/agents.py index 4d09e4d1e0f..14d4a11788b 100644 --- a/litellm/proxy/client/cli/commands/agents.py +++ b/litellm/proxy/client/cli/commands/agents.py @@ -10,11 +10,11 @@ from dataclasses import dataclass from itertools import chain from pathlib import Path from types import MappingProxyType -from typing import Final, TypeAlias, TypeVar +from typing import Final, Literal, TypeAlias, TypeVar import click import requests -from pydantic import JsonValue, TypeAdapter, ValidationError +from pydantic import ConfigDict, JsonValue, TypeAdapter, ValidationError from litellm.proxy.common_utils.codex_model_catalog import ( CODEX_BASE_INSTRUCTIONS_PATH, @@ -39,6 +39,7 @@ from .pi import ( LITELLM_PROXY_API_KEY_ENV, PI_PROVIDER_NAME, ListingFailure, + ModelLimits, PiSyncError, fetch_model_ids, fetch_model_limits, @@ -502,9 +503,25 @@ class _OpenCodeLimit(LiteLLMBaseModel): output: int +class _OpenCodeVariant(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + reasoningEffort: str + + +class _OpenCodeReasoningOption(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + type: Literal["effort"] = "effort" + values: tuple[str, ...] + + class _OpenCodeModel(LiteLLMBaseModel): name: str limit: _OpenCodeLimit | None = None + reasoning: Literal[True] | None = None + reasoning_options: tuple[_OpenCodeReasoningOption, ...] | None = None + variants: Mapping[str, _OpenCodeVariant] | None = None class _OpenCodeProviderOptions(LiteLLMBaseModel): @@ -523,15 +540,43 @@ class _OpenCodeConfig(LiteLLMBaseModel): provider: Mapping[str, _OpenCodeProvider] -def _opencode_model_entry(model: ListedModel) -> _OpenCodeModel: - if model.max_input_tokens is None or model.max_output_tokens is None: - return _OpenCodeModel(name=model.id) +_NO_GROUPS: Final[Mapping[str, ModelLimits]] = MappingProxyType({}) + + +def _reasoning_efforts(model_id: str, groups: Mapping[str, ModelLimits]) -> tuple[str, ...] | None: + group: Final = groups.get(model_id) + return group.supported_reasoning_efforts if group is not None else None + + +def _opencode_model_entry(model: ListedModel, groups: Mapping[str, ModelLimits]) -> _OpenCodeModel: + """One OpenCode model entry, with the model group's reasoning efforts as its variants when known. + + OpenCode 1.x builds its own variants for a reasoning model and replaces them only through + `reasoning_options`. 2.x ignores `reasoning` and `reasoning_options` and builds its own variants + unless the entry lists some, so the efforts go in all three, and a model with none lists an + empty `variants`.""" + limit: Final = ( + _OpenCodeLimit(context=model.max_input_tokens, output=model.max_output_tokens) + if model.max_input_tokens is not None and model.max_output_tokens is not None + else None + ) + efforts: Final = _reasoning_efforts(model.id, groups) + if efforts is None: + return _OpenCodeModel(name=model.id, limit=limit) + if not efforts: + return _OpenCodeModel(name=model.id, limit=limit, variants=MappingProxyType({})) return _OpenCodeModel( - name=model.id, limit=_OpenCodeLimit(context=model.max_input_tokens, output=model.max_output_tokens) + name=model.id, + limit=limit, + reasoning=True, + reasoning_options=(_OpenCodeReasoningOption(values=efforts),), + variants=MappingProxyType({effort: _OpenCodeVariant(reasoningEffort=effort) for effort in efforts}), ) -def opencode_provider_config(base_url: str, models: Sequence[ListedModel]) -> str: +def opencode_provider_config( + base_url: str, models: Sequence[ListedModel], groups: Mapping[str, ModelLimits] = _NO_GROUPS +) -> str: """OPENCODE_CONFIG_CONTENT declaring the proxy as OpenCode provider `litellm`. One model entry per chat-capable /v1/models row (mode chat, responses, or @@ -539,6 +584,7 @@ def opencode_provider_config(base_url: str, models: Sequence[ListedModel]) -> st is read back through {env:OPENAI_API_KEY}, which build_agent_env exports, so it never lands in the config text. OpenCode merges this inline config over the user's own files, leaving unrelated keys and providers untouched. + `groups` is /model_group/info by model group, where the reasoning efforts come from. """ chat_models: Final = _chat_models(models) provider: Final = _OpenCodeProvider( @@ -548,7 +594,7 @@ def opencode_provider_config(base_url: str, models: Sequence[ListedModel]) -> st baseURL=base_url.rstrip("/") + "/v1", apiKey=f"{{env:{OPENAI_API_KEY_ENV}}}", ), - models=MappingProxyType({m.id: _opencode_model_entry(m) for m in chat_models}), + models=MappingProxyType({m.id: _opencode_model_entry(m, groups) for m in chat_models}), ) config: Final = _OpenCodeConfig(provider=MappingProxyType({OPENCODE_PROVIDER_ID: provider})) return config.model_dump_json(exclude_none=True) @@ -563,7 +609,8 @@ def opencode_model_sync_env( ) -> Mapping[str, str] | ModelSyncSkipped: """Env addition that hands OpenCode the proxy's model list, or why it was skipped. - Fetches /v1/models with the key and packs it into OPENCODE_CONFIG_CONTENT. + Fetches /v1/models with the key, and /model_group/info for each model's reasoning + efforts, and packs them into OPENCODE_CONFIG_CONTENT. An OPENCODE_CONFIG_CONTENT already in the environment is left alone, and a failed fetch is reported rather than raised: OpenCode still launches on the plain OPENAI_* env, just without a synced model list. @@ -573,11 +620,16 @@ def opencode_model_sync_env( listing: Final = _fetch_model_listing(base_url, api_key, get=get) if isinstance(listing, ModelSyncSkipped): return listing - return MappingProxyType({OPENCODE_CONFIG_CONTENT_ENV: opencode_provider_config(base_url, listing)}) + groups: Final = fetch_model_limits(base_url, api_key, get=get) + return MappingProxyType({OPENCODE_CONFIG_CONTENT_ENV: opencode_provider_config(base_url, listing, groups)}) def codex_model_catalog( - models: Sequence[ListedModel], stock: Sequence[CodexStockModel], instructions: str + models: Sequence[ListedModel], + stock: Sequence[CodexStockModel], + instructions: str, + groups: Mapping[str, ModelLimits] = _NO_GROUPS, + client_version: str | None = None, ) -> str | None: """The `model_catalog_json` body listing the proxy's chat models, or None if there are none. @@ -588,12 +640,26 @@ def codex_model_catalog( Codex hides it or keeps it off the API, and keeps Codex's upgrade nudge only when the model it points at is served too. A model Codex does not know gets the fallback entry, with the same base instructions Codex itself uses so the agent never - runs without a system prompt. The catalog is the one `/v1/models?client_version=...` - serves, built from the installed Codex's own entries instead of the proxy's vendored ones. + runs without a system prompt, and offers the reasoning efforts `groups` gives its model group + that the installed Codex, at `client_version`, can parse. The catalog is the one + `/v1/models?client_version=...` serves, built from the installed Codex's own entries instead of + the proxy's vendored ones. """ - rows: Final = tuple(CodexCatalogRow(id=m.id, mode=m.mode, max_input_tokens=m.max_input_tokens) for m in models) + rows: Final = tuple( + CodexCatalogRow( + id=m.id, + mode=m.mode, + max_input_tokens=m.max_input_tokens, + supported_reasoning_efforts=_reasoning_efforts(m.id, groups), + ) + for m in models + ) body: Final = codex_models_response_json( - rows, stock=MappingProxyType({m.slug: m for m in stock}), instructions=instructions, byte_limit=None + rows, + stock=MappingProxyType({m.slug: m for m in stock}), + instructions=instructions, + byte_limit=None, + client_version=client_version, ) return body.json if body.listed else None @@ -662,6 +728,25 @@ def _stock_codex_models( return ModelSyncSkipped(f"`{name} debug models` printed no model catalog: {e.errors()[0]['msg']}") +def _codex_version( + binary: str, env: Mapping[str, str], *, run: Callable[..., subprocess.CompletedProcess[str]] +) -> str | None: + """The version `codex --version` prints (`codex-cli 0.165.0`), or None when it cannot be read.""" + try: + completed: Final = run( + _windows_command(binary, (binary, "--version")), + env=dict(env), + stdin=subprocess.DEVNULL, + capture_output=True, + encoding="utf-8", + timeout=_CODEX_PREFLIGHT_TIMEOUT_SECONDS, + ) + except (OSError, subprocess.TimeoutExpired): + return None + words: Final = completed.stdout.split() + return words[-1] if completed.returncode == 0 and words else None + + def codex_model_sync_args( base_env: Mapping[str, str], base_url: str, @@ -678,8 +763,9 @@ def codex_model_sync_args( Codex has no env or inline equivalent of OPENCODE_CONFIG_CONTENT: the catalog must be a file, so it is written under $CODEX_HOME (default ~/.codex) and atomically replaced on every launch. The Codex at `binary` first lists its - own models, so the ones the proxy serves keep that Codex's entries, and then - reads the file back once before it is handed over. The key never lands in + own models, so the ones the proxy serves keep that Codex's entries, and its + version, so a model it does not know offers only reasoning efforts it can + parse, and then reads the file back once before it is handed over. The key never lands in the file. A failed fetch, read, listing, write or read-back is reported rather than raised: Codex still launches with its built-in catalog and takes a proxy model by name via -m, and a rejected file stays on disk to be looked @@ -700,7 +786,9 @@ def codex_model_sync_args( stock: Final = _stock_codex_models(binary, base_env, run=run) if isinstance(stock, ModelSyncSkipped): return stock - catalog: Final = codex_model_catalog(listing, stock, instructions) + groups: Final = fetch_model_limits(base_url, api_key, get=get) + version: Final = _codex_version(binary, base_env, run=run) + catalog: Final = codex_model_catalog(listing, stock, instructions, groups, version) if catalog is None: return ModelSyncSkipped(f"{base_url.rstrip('/')}/v1/models lists no chat models") try: diff --git a/litellm/proxy/client/cli/commands/pi.py b/litellm/proxy/client/cli/commands/pi.py index 357b82bdf67..f02395b98ef 100644 --- a/litellm/proxy/client/cli/commands/pi.py +++ b/litellm/proxy/client/cli/commands/pi.py @@ -50,8 +50,11 @@ class PiSyncError: @dataclass(frozen=True, slots=True) class ModelLimits: + """What /model_group/info says about one model group; `supported_reasoning_efforts` is None when unknown.""" + context_window: int | None max_tokens: int | None + supported_reasoning_efforts: tuple[str, ...] | None = None _NonEmptyString = Annotated[str, StringConstraints(min_length=1)] @@ -79,6 +82,7 @@ class _ModelGroup(LiteLLMBaseModel): model_group: str max_input_tokens: float | None = None max_output_tokens: float | None = None + supported_reasoning_efforts: tuple[str, ...] | None = None class _ModelGroupList(LiteLLMBaseModel): @@ -137,8 +141,8 @@ def fetch_model_limits( *, get: Callable[..., requests.Response] = requests.get, ) -> Mapping[str, ModelLimits]: - """Best effort: pi falls back to its own defaults for models without limits, - so an unavailable /model_group/info must not block the launch.""" + """Best effort: pi, OpenCode and Codex fall back to their own defaults for models without + limits or reasoning efforts, so an unavailable /model_group/info must not block the launch.""" url: Final = base_url.rstrip("/") + "/model_group/info" try: resp: Final = get( @@ -156,6 +160,7 @@ def fetch_model_limits( group.model_group: ModelLimits( context_window=int(group.max_input_tokens) if group.max_input_tokens else None, max_tokens=int(group.max_output_tokens) if group.max_output_tokens else None, + supported_reasoning_efforts=group.supported_reasoning_efforts, ) for group in listing.data } @@ -168,6 +173,27 @@ def models_json_path(env: Mapping[str, str]) -> Path: return root / "models.json" +_PI_LADDER_LEVELS: Final = ("minimal", "low", "medium", "high") +_PI_OPT_IN_LEVELS: Final = ("xhigh", "max") +_REASONING_OFF_EFFORT: Final = "none" + + +def _thinking_level_map(efforts: tuple[str, ...]) -> dict[str, JsonValue]: # mutable-ok: JSON object is serialized + """Each level pi offers maps to a value: pi drops a level mapped to null and shows xhigh and max only when set.""" + return { + "off": _REASONING_OFF_EFFORT if _REASONING_OFF_EFFORT in efforts else None, + **{level: level if level in efforts else None for level in _PI_LADDER_LEVELS}, + **{level: level for level in _PI_OPT_IN_LEVELS if level in efforts}, + } + + +def _reasoning_fields(limit: ModelLimits | None) -> Mapping[str, JsonValue]: + efforts: Final = limit.supported_reasoning_efforts if limit else None + if not efforts: + return MappingProxyType({}) + return MappingProxyType({"reasoning": True, "thinkingLevelMap": _thinking_level_map(efforts)}) + + def _model_entry( model_id: str, limits: Mapping[str, ModelLimits] ) -> dict[str, JsonValue]: # mutable-ok: JSON object is serialized @@ -178,7 +204,7 @@ def _model_entry( output: Final[dict[str, JsonValue]] = ( # mutable-ok: JSON field {"maxTokens": limit.max_tokens} if limit and limit.max_tokens else {} ) - return {"id": model_id, **context, **output} + return {"id": model_id, **context, **output, **_reasoning_fields(limit)} def provider_block( diff --git a/litellm/proxy/common_utils/codex_model_catalog.py b/litellm/proxy/common_utils/codex_model_catalog.py index 9b42f99305e..0d005efdf7d 100644 --- a/litellm/proxy/common_utils/codex_model_catalog.py +++ b/litellm/proxy/common_utils/codex_model_catalog.py @@ -7,7 +7,8 @@ never the OpenAI list shape. This module builds that body from the rows ``/v1/mo lists: a model Codex knows keeps Codex's own entry for it, any other model gets the fallback entry Codex itself uses for an unknown slug, and ``model_info.service_tiers`` becomes the entry's ``service_tiers``, which Codex turns into slash commands (``/ultrafast``) that send ``service_tier`` -upstream. +upstream. A fallback entry offers the model group's ``supported_reasoning_efforts`` as its reasoning +levels, trimmed to the ones the requesting Codex release can parse. The stock entries are Codex 0.159.3's bundled catalog, vendored unchanged from https://github.com/openai/codex/blob/rust-v0.159.3/codex-rs/models-manager/models.json (Apache-2.0) @@ -26,9 +27,11 @@ from pathlib import Path from types import MappingProxyType, NoneType from typing import TYPE_CHECKING, Annotated, Final, Literal -from pydantic import ConfigDict, StringConstraints, TypeAdapter, ValidationError +from packaging.version import InvalidVersion, Version +from pydantic import ConfigDict, Field, StringConstraints, TypeAdapter, ValidationError from litellm._logging import verbose_proxy_logger +from litellm.router_utils.reasoning_effort_strength import nearest_declared_reasoning_effort from litellm.types.llms.base_model import LiteLLMBaseModel if TYPE_CHECKING: @@ -43,6 +46,9 @@ CODEX_CHAT_MODES: Final = frozenset({"chat", "responses"}) _TierId = Annotated[str, StringConstraints(strip_whitespace=True, min_length=1)] _BODY_PREFIX: Final = '{"models":[' _BODY_SUFFIX: Final = "]}" +_FIRST_CODEX_WITH_CUSTOM_EFFORTS: Final = Version("0.138.0") +_EFFORTS_EVERY_CODEX_PARSES: Final = frozenset({"none", "minimal", "low", "medium", "high", "xhigh"}) +_PREFERRED_DEFAULT_EFFORT: Final = "medium" class CodexServiceTier(LiteLLMBaseModel): @@ -67,6 +73,15 @@ class _ConfiguredTier(LiteLLMBaseModel): _CONFIGURED_TIERS: Final = TypeAdapter(tuple[_TierId | _ConfiguredTier, ...]) +class CodexReasoningLevel(LiteLLMBaseModel): + """A `ReasoningEffortPreset` as Codex reads it: picking it sends `effort` as the request's reasoning effort.""" + + model_config = ConfigDict(frozen=True) + + effort: str + description: str + + class CodexTruncationPolicy(LiteLLMBaseModel): mode: Literal["bytes"] = "bytes" limit: int = 10_000 @@ -85,7 +100,8 @@ class CodexFallbackModel(LiteLLMBaseModel): slug: str display_name: str description: NoneType = None - supported_reasoning_levels: tuple[()] = () + default_reasoning_level: str | None = Field(default=None, exclude_if=lambda level: level is None) + supported_reasoning_levels: tuple[CodexReasoningLevel, ...] = () shell_type: Literal["unified_exec"] = "unified_exec" visibility: Literal["list"] = "list" supported_in_api: Literal[True] = True @@ -142,7 +158,7 @@ class CodexCatalogRow: """What the catalog needs to know about one listed model: its public id, the listing's mode and input limit, the upstream model behind it, and the `model_info` values the operator set; `service_tiers` holds one raw `model_info.service_tiers` value per deployment behind the id, None - where a deployment sets none.""" + where a deployment sets none. `supported_reasoning_efforts` is the model group's, None when unknown.""" id: str mode: str | None = None @@ -150,6 +166,7 @@ class CodexCatalogRow: upstream_model: str | None = None display_name: str | None = None service_tiers: tuple[object, ...] = () + supported_reasoning_efforts: tuple[str, ...] | None = None @cache @@ -242,8 +259,36 @@ def configured_service_tiers( return offered -def _codex_can_drive(row: CodexCatalogRow) -> bool: - return "*" not in row.id and (row.mode is None or row.mode in CODEX_CHAT_MODES) +def _codex_can_drive(model_id: str, mode: str | None) -> bool: + return "*" not in model_id and (mode is None or mode in CODEX_CHAT_MODES) + + +def _parses_custom_efforts(client_version: str | None) -> bool: + try: + return client_version is not None and Version(client_version) >= _FIRST_CODEX_WITH_CUSTOM_EFFORTS + except InvalidVersion: + return False + + +def codex_reasoning_levels( + efforts: Sequence[str], client_version: str | None +) -> tuple[tuple[CodexReasoningLevel, ...], str | None]: + """The reasoning levels a fallback entry offers for a model group's efforts, and its default level. + + Codex before 0.138.0 fails the whole catalog on an effort it has no name for, so for it, and for + a version that does not parse, only the efforts every release knows are offered. The default is + set whenever a level is offered, because Codex reads a missing one as `none`: medium, else the + nearest stronger declared level, else the first one.""" + parses_any: Final = _parses_custom_efforts(client_version) + offered: Final = tuple(effort for effort in efforts if parses_any or effort in _EFFORTS_EVERY_CODEX_PARSES) + if not offered: + return (), None + nearest: Final = nearest_declared_reasoning_effort(_PREFERRED_DEFAULT_EFFORT, offered) + levels: Final = tuple( + CodexReasoningLevel(effort=effort, description=f"Sends reasoning_effort={effort} upstream") + for effort in offered + ) + return levels, nearest if nearest in offered else offered[0] def _stock_for(row: CodexCatalogRow, stock: Mapping[str, CodexStockModel]) -> CodexStockModel | None: @@ -260,6 +305,7 @@ def _entry( stock: Mapping[str, CodexStockModel], served: frozenset[str], instructions: str, + client_version: str | None, ) -> CodexCatalogEntry: stock_model: Final = _stock_for(row, stock) stock_tiers: Final = MappingProxyType( @@ -267,9 +313,12 @@ def _entry( ) configured: Final = configured_service_tiers(row.service_tiers, row.id, stock_tiers) if stock_model is None: + levels, default_level = codex_reasoning_levels(row.supported_reasoning_efforts or (), client_version) return CodexFallbackModel( slug=row.id, display_name=row.display_name if row.display_name is not None else row.id, + default_reasoning_level=default_level, + supported_reasoning_levels=levels, priority=index, service_tiers=configured if configured is not None else (), context_window=row.max_input_tokens, @@ -351,6 +400,7 @@ def codex_models_response_json( stock: Mapping[str, CodexStockModel] | None = None, instructions: str | None = None, byte_limit: int | None = CODEX_CATALOG_BYTE_LIMIT, + client_version: str | None = None, ) -> CodexCatalogBody: """Codex's `ModelsResponse` body for `rows`, in listing order, cut to Codex's byte limit. @@ -359,13 +409,16 @@ def codex_models_response_json( byte limit silently and keeps its bundled one, so the body holds only the entries that fit, the ones offering a service tier taken first and the rest in listing order, an entry too large for the bytes left passed over, every kept entry at its listing position; `byte_limit=None` - keeps every entry. + keeps every entry. `client_version` is the requesting Codex's version, which decides the reasoning + levels a fallback entry may offer; None counts as a version too old for custom levels. """ known: Final = bundled_codex_models() if stock is None else stock prompt: Final = codex_base_instructions() if instructions is None else instructions - drivable: Final = tuple(row for row in rows if _codex_can_drive(row)) + drivable: Final = tuple(row for row in rows if _codex_can_drive(row.id, row.mode)) served: Final = frozenset(row.id for row in drivable) - entries: Final = tuple(_entry(index, row, known, served, prompt) for index, row in enumerate(drivable)) + entries: Final = tuple( + _entry(index, row, known, served, prompt, client_version) for index, row in enumerate(drivable) + ) serialized: Final = tuple(_serialized(entry) for entry in entries) kept: Final = _kept_positions(entries, serialized, byte_limit) return CodexCatalogBody( @@ -375,6 +428,15 @@ def codex_models_response_json( ) +def _group_reasoning_efforts( + row: ModelInfoResponse, lookup_id: str, llm_router: Router | None, team_id: str | None +) -> tuple[str, ...] | None: + if llm_router is None or not _codex_can_drive(row["id"], row.get("mode")): + return None + group: Final = llm_router.get_model_group_info(lookup_id, include_hidden=True, team_id=team_id) + return group.supported_reasoning_efforts if group is not None else None + + def _catalog_row( row: ModelInfoResponse, lookup_id: str, label_id: str, llm_router: Router | None, team_id: str | None ) -> CodexCatalogRow: @@ -385,6 +447,7 @@ def _catalog_row( upstream_model=llm_router.get_routable_upstream_model(lookup_id, team_id) if llm_router is not None else None, display_name=llm_router.get_configured_display_name(label_id) if llm_router is not None else None, service_tiers=llm_router.get_configured_service_tiers(lookup_id, team_id) if llm_router is not None else (), + supported_reasoning_efforts=_group_reasoning_efforts(row, lookup_id, llm_router, team_id), ) @@ -399,7 +462,8 @@ def codex_catalog_rows( team-scoped rows resolve the way the Anthropic listing's display names do, a `model_group_alias` reading its target's deployments under its own name; `team_id` is the requesting key's team, so the upstream model that picks Codex's stock entry and the tiers are read only off the deployments its - requests can route to. Display names are read under `label_entries`' ids when given.""" + requests can route to, and the reasoning efforts are the ones `/model_group/info` reports for the + entry. Display names are read under `label_entries`' ids when given.""" lookup_ids: Final = MappingProxyType(dict(entries)) label_ids: Final = lookup_ids if label_entries is None else MappingProxyType(dict(label_entries)) return tuple( @@ -416,9 +480,13 @@ def codex_model_list_body( llm_router: Router | None, team_id: str | None = None, label_entries: Sequence[tuple[str, str]] | None = None, + *, + client_version: str | None = None, ) -> str: """The `/v1/models?client_version=...` body, logging the models Codex's byte limit left out.""" - body: Final = codex_models_response_json(codex_catalog_rows(rows, entries, llm_router, team_id, label_entries)) + body: Final = codex_models_response_json( + codex_catalog_rows(rows, entries, llm_router, team_id, label_entries), client_version=client_version + ) if body.left_out: verbose_proxy_logger.warning( "Codex model catalog cut at %d bytes, left out: %s. Models offering a service tier are kept first, " diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 64fb2791913..3d6a9bd3c40 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -12292,8 +12292,10 @@ async def model_list( - client_version: Sent by Codex CLI (`?client_version=0.159.3`) when it fetches a provider's model catalog. When present, the response is Codex's own catalog shape (`{"models": [...]}`) built from the same listing, with - each model's `model_info.service_tiers` as its service tiers; absent, - the OpenAI shape below is returned + each model's `model_info.service_tiers` as its service tiers and, for a + model that Codex release does not know, the reasoning efforts + `/model_group/info` reports that it can parse; absent, the OpenAI shape + below is returned - include_metadata: Include additional metadata in the response with fallback information - fallback_type: Type of fallbacks to include ("general", "context_window", "content_policy") Defaults to "general" when include_metadata=true @@ -12444,7 +12446,11 @@ async def model_list( if client_version is not None: return Response( content=codex_model_list_body( - admin_listing, admin_entries, llm_router, team_id or user_api_key_dict.team_id + admin_listing, + admin_entries, + llm_router, + team_id or user_api_key_dict.team_id, + client_version=client_version, ), media_type="application/json", ) @@ -12514,7 +12520,14 @@ async def model_list( listing: Final = cast(Sequence[ModelInfoResponse], model_data) # cast-ok: rows built above if client_version is not None: return Response( - content=codex_model_list_body(listing, entries, llm_router, team_id or user_api_key_dict.team_id, labels), + content=codex_model_list_body( + listing, + entries, + llm_router, + team_id or user_api_key_dict.team_id, + labels, + client_version=client_version, + ), media_type="application/json", ) diff --git a/litellm/router_utils/reasoning_effort_capability.py b/litellm/router_utils/reasoning_effort_capability.py index 16b8a22a40f..c802c7b37ec 100644 --- a/litellm/router_utils/reasoning_effort_capability.py +++ b/litellm/router_utils/reasoning_effort_capability.py @@ -27,9 +27,7 @@ it above. from collections.abc import Mapping, Sequence from dataclasses import dataclass from types import MappingProxyType -from typing import Final, Literal, TypeAlias, get_args - -from typing_extensions import assert_never +from typing import Final, TypeAlias, get_args import litellm from litellm._logging import verbose_logger @@ -37,10 +35,14 @@ from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.get_supported_openai_params import get_supported_openai_params from litellm.litellm_core_utils.reasoning_effort_utils import reasoning_effort_from_thinking from litellm.llms.anthropic.common_utils import AnthropicModelInfo, validated_object_mapping +from litellm.router_utils.reasoning_effort_strength import ( + REASONING_EFFORT_STRENGTH_RANK, + ReasoningEffortRounding, + nearest_declared_reasoning_effort, +) from litellm.types.llms.anthropic import AnthropicThinkingParam from litellm.types.llms.openai import REASONING_EFFORT -ReasoningEffortRounding: TypeAlias = Literal["up", "down"] RequestedReasoningEffort: TypeAlias = str | Mapping[str, object] | None REASONING_EFFORT_ADVERTISEMENT_ORDER: Final[tuple[str, ...]] = get_args(REASONING_EFFORT) @@ -115,35 +117,6 @@ def declared_reasoning_efforts_for_model(model: str, custom_llm_provider: str) - return declared_reasoning_efforts(entry) -REASONING_EFFORT_STRENGTH_ORDER: Final = ("minimal", "low", "medium", "high", "xhigh", "max") -_STRENGTH_RANK: Final = MappingProxyType({effort: rank for rank, effort in enumerate(REASONING_EFFORT_STRENGTH_ORDER)}) - - -def nearest_declared_reasoning_effort( - requested: str, declared: Sequence[str], rounding: ReasoningEffortRounding = "up" -) -> str: - """Rounding up takes the weakest declared level at least as strong as the request, else the - strongest declared, so the caller gets no less reasoning than it asked for instead of a rejected - call. Rounding down mirrors it: the strongest declared level at most as strong, else the - weakest. none is the off switch rather than a strength, so it is never rounded onto the ladder - and no level is rounded to it: a caller who turned reasoning off must not be billed for it, and - a model that cannot turn it off says so itself. A level outside the strength order is likewise - returned as is for upstream to judge.""" - ranked: Final = sorted( - (effort for effort in declared if effort in _STRENGTH_RANK), key=lambda effort: _STRENGTH_RANK[effort] - ) - if requested in ranked or requested not in _STRENGTH_RANK or not ranked: - return requested - strength: Final = _STRENGTH_RANK[requested] - match rounding: - case "up": - return next((effort for effort in ranked if _STRENGTH_RANK[effort] >= strength), ranked[-1]) - case "down": - return next((effort for effort in reversed(ranked) if _STRENGTH_RANK[effort] <= strength), ranked[0]) - case _: - assert_never(rounding) - - _ROUNDING_KEY: Final = "reasoning_effort_rounding" _DEFAULT_EFFORT: Final = "default" _EFFORT_KEY: Final = "effort" @@ -163,7 +136,7 @@ def deployment_reasoning_effort_levels(deployment_model_info: object) -> tuple[s def deployment_rounds_reasoning_effort(deployment_model_info: object) -> bool: declared: Final = deployment_reasoning_effort_levels(deployment_model_info) - return declared is not None and any(level in _STRENGTH_RANK for level in declared) + return declared is not None and any(level in REASONING_EFFORT_STRENGTH_RANK for level in declared) def with_adaptive_thinking_as_effort( diff --git a/litellm/router_utils/reasoning_effort_strength.py b/litellm/router_utils/reasoning_effort_strength.py new file mode 100644 index 00000000000..f7b51b75a9f --- /dev/null +++ b/litellm/router_utils/reasoning_effort_strength.py @@ -0,0 +1,45 @@ +"""How strong each reasoning_effort level is, and rounding a requested level onto a declared set. + +Imports nothing from the SDK, so the `lite` CLI, which loads without initializing litellm, can use it. +""" + +from collections.abc import Sequence +from types import MappingProxyType +from typing import Final, Literal, TypeAlias + +from typing_extensions import assert_never + +ReasoningEffortRounding: TypeAlias = Literal["up", "down"] + +REASONING_EFFORT_STRENGTH_ORDER: Final = ("minimal", "low", "medium", "high", "xhigh", "max") +REASONING_EFFORT_STRENGTH_RANK: Final = MappingProxyType( + {effort: rank for rank, effort in enumerate(REASONING_EFFORT_STRENGTH_ORDER)} +) + + +def nearest_declared_reasoning_effort( + requested: str, declared: Sequence[str], rounding: ReasoningEffortRounding = "up" +) -> str: + """Rounding up takes the weakest declared level at least as strong as the request, else the + strongest declared, so the caller gets no less reasoning than it asked for instead of a rejected + call. Rounding down mirrors it: the strongest declared level at most as strong, else the + weakest. none is the off switch rather than a strength, so it is never rounded onto the ladder + and no level is rounded to it: a caller who turned reasoning off must not be billed for it, and + a model that cannot turn it off says so itself. A level outside the strength order is likewise + returned as is for upstream to judge.""" + ranked: Final = sorted( + (effort for effort in declared if effort in REASONING_EFFORT_STRENGTH_RANK), + key=lambda effort: REASONING_EFFORT_STRENGTH_RANK[effort], + ) + if requested in ranked or requested not in REASONING_EFFORT_STRENGTH_RANK or not ranked: + return requested + strength: Final = REASONING_EFFORT_STRENGTH_RANK[requested] + match rounding: + case "up": + return next((effort for effort in ranked if REASONING_EFFORT_STRENGTH_RANK[effort] >= strength), ranked[-1]) + case "down": + return next( + (effort for effort in reversed(ranked) if REASONING_EFFORT_STRENGTH_RANK[effort] <= strength), ranked[0] + ) + case _: + assert_never(rounding) diff --git a/tests/unit/proxy/client/cli/test_agents.py b/tests/unit/proxy/client/cli/test_agents.py index 3c79b916202..8df08da1228 100644 --- a/tests/unit/proxy/client/cli/test_agents.py +++ b/tests/unit/proxy/client/cli/test_agents.py @@ -59,6 +59,9 @@ class _FakeResponse: self.status_code = status_code self.content = json.dumps(body).encode() if body is not None else b"" + def json(self): + return json.loads(self.content) + class _Recorder: def __init__(self, returns=None): @@ -158,24 +161,47 @@ _STOCK_CATALOG = json.dumps({"models": list(_STOCK_MODELS.values())}) class _FakeRun: - """A `codex` that prints `stock` from a bare `debug models` and answers a catalog override with `returncode`. + """A `codex` that prints `stock` from a bare `debug models`, `version` from `--version`, and answers a + catalog override with `returncode`. `stock=None` is a Codex with no `debug models` at all: every call answers with `returncode` and `stderr`. + `version=None` is a `--version` that fails, an exception one that cannot run. """ - def __init__(self, returncode=0, stderr="", stock=_STOCK_CATALOG): + def __init__(self, returncode=0, stderr="", stock=_STOCK_CATALOG, version="codex-cli 0.165.0"): self.returncode = returncode self.stderr = stderr self.stock = stock + self.version = version self.calls = [] def __call__(self, args, **kwargs): self.calls.append((args, kwargs)) + if self.stock is not None and "--version" in str(args): + if isinstance(self.version, Exception): + raise self.version + if self.version is None: + return subprocess.CompletedProcess(args, 2, "", "error: unexpected argument '--version'") + return subprocess.CompletedProcess(args, 0, f"{self.version}\n", "") if self.stock is not None and "model_catalog_json=" not in str(args): return subprocess.CompletedProcess(args, 0, self.stock, "") return subprocess.CompletedProcess(args, self.returncode, "", self.stderr) +def _routed_get(captured, listing, groups): + """A proxy answering /model_group/info with `groups` (or `groups` as the status it fails with) and any + other URL with `listing`.""" + + def fake_get(url, headers, timeout): + captured.setdefault("urls", []).append(url) + captured["headers"] = headers + if not url.endswith("/model_group/info"): + return _FakeResponse(200, listing) + return _FakeResponse(groups) if isinstance(groups, int) else _FakeResponse(200, {"data": list(groups)}) + + return fake_get + + class _FakeJsonResponse: def __init__(self, status_code, payload=None): self.status_code = status_code @@ -532,15 +558,9 @@ class TestOpencodeModelSync: def _listing(*models): return {"object": "list", "data": list(models)} - def _sync(self, listing, base_env=None, base_url="http://localhost:4000/"): + def _sync(self, listing, base_env=None, base_url="http://localhost:4000/", groups=()): captured = {} - - def fake_get(url, headers, timeout): - captured["url"] = url - captured["headers"] = headers - return _FakeResponse(200, listing) - - env = opencode_model_sync_env(base_env or {}, base_url, "sk-key", get=fake_get) + env = opencode_model_sync_env(base_env or {}, base_url, "sk-key", get=_routed_get(captured, listing, groups)) return captured, env def test_declares_proxy_as_litellm_provider_with_listed_models(self): @@ -548,9 +568,9 @@ class TestOpencodeModelSync: {"id": "gpt-5.5", "object": "model", "created": 1, "owned_by": "openai", "mode": "chat"}, {"id": "claude-opus-4-7", "object": "model", "created": 1, "owned_by": "openai"}, ) - captured, env = self._sync(listing) + captured, env = self._sync(listing, groups=({"model_group": "gpt-5.5", "supported_reasoning_efforts": None},)) - assert captured["url"] == "http://localhost:4000/v1/models" + assert captured["urls"] == ["http://localhost:4000/v1/models", "http://localhost:4000/model_group/info"] assert captured["headers"] == {"Authorization": "Bearer sk-key"} config = json.loads(env["OPENCODE_CONFIG_CONTENT"]) provider = config["provider"]["litellm"] @@ -583,6 +603,56 @@ class TestOpencodeModelSync: assert models["gpt-5.5"]["limit"] == {"context": 400000, "output": 128000} assert "limit" not in models["half"] + @pytest.mark.parametrize( + ("efforts", "reasoning"), + [ + ( + ["low", "high", "max"], + { + "reasoning": True, + "reasoning_options": [{"type": "effort", "values": ["low", "high", "max"]}], + "variants": { + "low": {"reasoningEffort": "low"}, + "high": {"reasoningEffort": "high"}, + "max": {"reasoningEffort": "max"}, + }, + }, + ), + ( + ["none", "turbo"], + { + "reasoning": True, + "reasoning_options": [{"type": "effort", "values": ["none", "turbo"]}], + "variants": {"none": {"reasoningEffort": "none"}, "turbo": {"reasoningEffort": "turbo"}}, + }, + ), + ([], {"variants": {}}), + (None, {}), + ], + ids=["declared", "unknown-names", "non-reasoning", "unknown"], + ) + def test_model_group_reasoning_efforts_become_exactly_the_opencode_variants(self, efforts, reasoning): + listing = self._listing( + { + "id": "m", + "object": "model", + "created": 1, + "owned_by": "openai", + "max_input_tokens": 9, + "max_output_tokens": 3, + } + ) + _, env = self._sync(listing, groups=({"model_group": "m", "supported_reasoning_efforts": efforts},)) + models = json.loads(env["OPENCODE_CONFIG_CONTENT"])["provider"]["litellm"]["models"] + assert models == {"m": {"name": "m", "limit": {"context": 9, "output": 3}, **reasoning}} + + def test_an_unavailable_model_group_info_keeps_the_listing_only_config(self): + listing = self._listing({"id": "m", "object": "model", "created": 1, "owned_by": "openai"}) + _, failed = self._sync(listing, groups=500) + _, unknown = self._sync(listing, groups=({"model_group": "m", "supported_reasoning_efforts": None},)) + assert failed == unknown + assert json.loads(failed["OPENCODE_CONFIG_CONTENT"])["provider"]["litellm"]["models"] == {"m": {"name": "m"}} + def test_non_chat_models_are_left_out(self): listing = self._listing( {"id": "chat", "object": "model", "created": 1, "owned_by": "openai", "mode": "chat"}, @@ -669,19 +739,13 @@ class TestCodexModelSync: def _row(model_id, **extra): return {"id": model_id, "object": "model", "created": 1, "owned_by": "openai", **extra} - def _sync(self, listing, codex_home, base_url="http://localhost:4000/", run=None): + def _sync(self, listing, codex_home, base_url="http://localhost:4000/", run=None, groups=()): captured = {} - - def fake_get(url, headers, timeout): - captured["url"] = url - captured["headers"] = headers - return _FakeResponse(200, listing) - result = codex_model_sync_args( {"CODEX_HOME": str(codex_home)}, base_url, "sk-key", - get=fake_get, + get=_routed_get(captured, listing, groups), run=_FakeRun() if run is None else run, ) return captured, result @@ -699,7 +763,7 @@ class TestCodexModelSync: listing = self._listing(self._row("gpt-5.5", mode="chat"), self._row("claude-opus-4-7")) captured, result = self._sync(listing, tmp_path / "codex") - assert captured["url"] == "http://localhost:4000/v1/models" + assert captured["urls"] == ["http://localhost:4000/v1/models", "http://localhost:4000/model_group/info"] assert captured["headers"] == {"Authorization": "Bearer sk-key"} path = self._catalog_path(result) assert path == str(tmp_path / "codex" / "litellm-models.json") @@ -776,6 +840,52 @@ class TestCodexModelSync: assert nullable in entry and entry[nullable] is None assert entry["base_instructions"].startswith("You are a coding agent running in the Codex CLI") + @pytest.mark.parametrize( + ("version", "levels"), + [ + ("codex-cli 0.165.0", ["low", "high", "max"]), + ("codex-cli 0.138.0", ["low", "high", "max"]), + ("codex-cli 0.137.0", ["low", "high"]), + ("codex-cli 0.138.0-alpha.3", ["low", "high"]), + ("codex-cli", ["low", "high"]), + (None, ["low", "high"]), + (OSError("codex vanished"), ["low", "high"]), + ], + ids=["current", "first-custom", "pre-custom", "pre-release", "no-number", "failed", "unrunnable"], + ) + def test_unknown_model_offers_the_group_efforts_the_installed_codex_parses(self, tmp_path, version, levels): + _, result = self._sync( + self._listing(self._row("m")), + tmp_path, + run=_FakeRun(version=version), + groups=({"model_group": "m", "supported_reasoning_efforts": ["low", "high", "max"]},), + ) + + assert isinstance(result, ModelSyncArgs) + entry = self._entries(tmp_path)["m"] + assert [level["effort"] for level in entry["supported_reasoning_levels"]] == levels + assert entry["supported_reasoning_levels"][0] == { + "effort": "low", + "description": "Sends reasoning_effort=low upstream", + } + assert entry["default_reasoning_level"] == "high" + + def test_known_models_and_unknown_or_empty_groups_keep_their_entries(self, tmp_path): + listing = self._listing(self._row("gpt-5.5"), self._row("unknown"), self._row("empty")) + groups = ( + {"model_group": "gpt-5.5", "supported_reasoning_efforts": ["max"]}, + {"model_group": "unknown", "supported_reasoning_efforts": None}, + {"model_group": "empty", "supported_reasoning_efforts": []}, + ) + self._sync(listing, tmp_path, groups=groups) + with_groups = self._entries(tmp_path) + self._sync(listing, tmp_path, groups=500) + + assert with_groups == self._entries(tmp_path) + assert with_groups["gpt-5.5"] == {**_STOCK_MODELS["gpt-5.5"], "priority": 0} + assert [with_groups[slug]["supported_reasoning_levels"] for slug in ("unknown", "empty")] == [[], []] + assert all("default_reasoning_level" not in with_groups[slug] for slug in ("unknown", "empty")) + def test_context_window_comes_from_max_input_tokens_for_unknown_models_only(self, tmp_path): listing = self._listing( self._row("big", max_input_tokens=400000), @@ -890,7 +1000,7 @@ class TestCodexModelSync: run=run, ) assert self._catalog_path(result) == str(tmp_path / "litellm-models.json") - assert len(run.calls) == 2 + assert len(run.calls) == 3 assert all(binary in command for command, _ in run.calls) def test_opencode_dispatch_never_runs_codex(self): @@ -915,6 +1025,7 @@ class TestCodexModelSync: assert [command for command, _ in run.calls] == [ ("codex", "debug", "models"), + ("codex", "--version"), ("codex", "-c", f"model_catalog_json={json.dumps(path)}", "debug", "models"), ] for _, options in run.calls: @@ -979,6 +1090,7 @@ class TestCodexModelSync: doubled = override.replace('"', '""') assert [command for command, _ in run.calls] == [ f'{_CMD_PREFIX}""{shim}" "debug" "models""', + f'{_CMD_PREFIX}""{shim}" "--version""', f'{_CMD_PREFIX}""{shim}" "-c" "{doubled}" "debug" "models""', ] diff --git a/tests/unit/proxy/client/cli/test_pi.py b/tests/unit/proxy/client/cli/test_pi.py index e6669564358..ae485914507 100644 --- a/tests/unit/proxy/client/cli/test_pi.py +++ b/tests/unit/proxy/client/cli/test_pi.py @@ -152,16 +152,25 @@ class TestFetchModelLimits: 200, { "data": [ - {"model_group": "m-a", "max_input_tokens": 131072, "max_output_tokens": 8192}, + { + "model_group": "m-a", + "max_input_tokens": 131072, + "max_output_tokens": 8192, + "supported_reasoning_efforts": ["low", "high", "max"], + }, {"model_group": "m-b", "max_input_tokens": None, "max_output_tokens": None}, + {"model_group": "m-c", "supported_reasoning_efforts": []}, ] }, ) limits = fetch_model_limits("http://localhost:4000/", "sk-key", get=fake_get) assert captured["url"] == "http://localhost:4000/model_group/info" - assert limits["m-a"] == ModelLimits(context_window=131072, max_tokens=8192) - assert limits["m-b"] == ModelLimits(context_window=None, max_tokens=None) + assert limits["m-a"] == ModelLimits( + context_window=131072, max_tokens=8192, supported_reasoning_efforts=("low", "high", "max") + ) + assert limits["m-b"] == ModelLimits(context_window=None, max_tokens=None, supported_reasoning_efforts=None) + assert limits["m-c"].supported_reasoning_efforts == () def test_non_200_degrades_to_no_limits(self): assert fetch_model_limits("http://localhost:4000", "sk-key", get=lambda *a, **k: _FakeResponse(403)) == {} @@ -219,6 +228,60 @@ class TestProviderBlock: {"id": "m-2"}, ] + @pytest.mark.parametrize( + ("efforts", "thinking_level_map"), + [ + ( + ("low", "medium", "high", "xhigh", "max"), + { + "off": None, + "minimal": None, + "low": "low", + "medium": "medium", + "high": "high", + "xhigh": "xhigh", + "max": "max", + }, + ), + ( + ("none", "low", "high", "max"), + {"off": "none", "minimal": None, "low": "low", "medium": None, "high": "high", "max": "max"}, + ), + ( + ("none", "minimal", "low", "medium", "high"), + {"off": "none", "minimal": "minimal", "low": "low", "medium": "medium", "high": "high"}, + ), + ( + ("high", "turbo"), + {"off": None, "minimal": None, "low": None, "medium": None, "high": "high"}, + ), + ], + ) + def test_reasoning_efforts_become_pi_thinking_levels(self, efforts, thinking_level_map): + block = provider_block( + "http://localhost:4000", + ("m-1",), + {"m-1": ModelLimits(context_window=131072, max_tokens=8192, supported_reasoning_efforts=efforts)}, + ) + assert block["models"] == [ + { + "id": "m-1", + "contextWindow": 131072, + "maxTokens": 8192, + "reasoning": True, + "thinkingLevelMap": thinking_level_map, + } + ] + + @pytest.mark.parametrize("efforts", [None, ()]) + def test_unknown_or_no_reasoning_efforts_leave_the_entry_as_it_was(self, efforts): + block = provider_block( + "http://localhost:4000", + ("m-1",), + {"m-1": ModelLimits(context_window=131072, max_tokens=8192, supported_reasoning_efforts=efforts)}, + ) + assert block["models"] == [{"id": "m-1", "contextWindow": 131072, "maxTokens": 8192}] + class TestSyncModelsJson: def test_creates_file_and_parent_dirs(self, tmp_path): diff --git a/tests/unit/proxy/common_utils/test_codex_model_catalog.py b/tests/unit/proxy/common_utils/test_codex_model_catalog.py index c9d4127c6d4..b23b2df15bd 100644 --- a/tests/unit/proxy/common_utils/test_codex_model_catalog.py +++ b/tests/unit/proxy/common_utils/test_codex_model_catalog.py @@ -113,6 +113,85 @@ def test_unknown_model_entry_carries_every_field_codex_deserializes_without_a_de assert (entry["context_window"], entry["base_instructions"], entry["service_tiers"]) == (272000, _PROMPT, []) +_NO_REASONING_BODY = ( + '{"models":[{"slug":"gpt-6-astra","display_name":"gpt-6-astra","description":null,' + '"supported_reasoning_levels":[],"shell_type":"unified_exec","visibility":"list","supported_in_api":true,' + '"priority":0,"service_tiers":[],"availability_nux":null,"upgrade":null,"support_verbosity":false,' + '"supports_reasoning_summaries":false,"supports_parallel_tool_calls":false,"default_verbosity":null,' + '"apply_patch_tool_type":null,"truncation_policy":{"mode":"bytes","limit":10000},' + '"experimental_supported_tools":[],"context_window":272000,"base_instructions":"base prompt"}]}' +) + + +@pytest.mark.parametrize("efforts", [None, ()], ids=["unknown", "non-reasoning"]) +@pytest.mark.parametrize("client_version", ["0.165.0", "0.120.0", None]) +def test_unknown_or_no_reasoning_efforts_keep_the_fallback_entry_byte_for_byte(efforts, client_version): + """The body an unknown model got before reasoning efforts reached the catalog.""" + assert ( + _body(_row("gpt-6-astra", supported_reasoning_efforts=efforts), client_version=client_version).json + == _NO_REASONING_BODY + ) + + +_ABSENT = object() + + +@pytest.mark.parametrize( + ("efforts", "client_version", "levels", "default"), + [ + (("low", "high", "max"), "0.165.0", ["low", "high", "max"], "high"), + ( + ("none", "low", "medium", "high", "xhigh", "max"), + "0.165.0", + ["none", "low", "medium", "high", "xhigh", "max"], + "medium", + ), + (("none",), "0.165.0", ["none"], "none"), + (("low", "turbo"), "0.165.0", ["low", "turbo"], "low"), + (("low", "high", "max"), "0.138.0", ["low", "high", "max"], "high"), + (("low", "high", "max"), "0.137.9", ["low", "high"], "high"), + (("low", "high", "max"), "0.138.0-alpha.2", ["low", "high"], "high"), + (("minimal", "max"), "not-a-version", ["minimal"], "minimal"), + (("low", "turbo"), None, ["low"], "low"), + (("max",), "0.137.0", [], _ABSENT), + ], +) +def test_unknown_model_offers_the_group_efforts_its_codex_release_parses(efforts, client_version, levels, default): + (entry,) = _models(_row("gpt-6-astra", supported_reasoning_efforts=efforts), client_version=client_version) + + assert [level["effort"] for level in entry["supported_reasoning_levels"]] == levels + assert all( + level["description"] == f"Sends reasoning_effort={level['effort']} upstream" + for level in entry["supported_reasoning_levels"] + ) + assert entry.get("default_reasoning_level", _ABSENT) == default + + +def test_a_model_codex_knows_keeps_its_own_reasoning_levels(): + stock = CodexStockModel( + slug="gpt-5.5", + display_name="GPT-5.5", + priority=3, + visibility="list", + default_reasoning_level="medium", + supported_reasoning_levels=[{"effort": "medium", "description": "Codex's own medium"}], + ) + + (entry,) = json.loads( + codex_models_response_json( + (_row("gpt-5.5", supported_reasoning_efforts=("low", "max")),), + stock={"gpt-5.5": stock}, + instructions=_PROMPT, + client_version="0.165.0", + ).json + )["models"] + + assert (entry["default_reasoning_level"], entry["supported_reasoning_levels"]) == ( + "medium", + [{"effort": "medium", "description": "Codex's own medium"}], + ) + + def test_known_model_keeps_codex_stock_entry_listed_under_the_served_id(): (entry,) = _models(_row("gpt-daybreak")) @@ -429,6 +508,9 @@ def test_catalog_rows_read_the_router_by_each_entry_lookup_id(): upstream_model="openai/gpt-5.5", display_name="Team 5.5", service_tiers=(["ultrafast"], None), + supported_reasoning_efforts=router.get_model_group_info( + "model_name_team-1_c0ffee" + ).supported_reasoning_efforts, ), CodexCatalogRow(id="plain", upstream_model="openai/some-unmapped-model", service_tiers=(None,)), ) @@ -452,7 +534,12 @@ def test_catalog_rows_read_an_alias_off_its_target_under_the_alias_name(): (row,) = codex_catalog_rows(listing, (("gpt-6", "gpt-6"),), router) assert row == CodexCatalogRow( - id="gpt-6", mode="chat", max_input_tokens=9, upstream_model="openai/gpt-6-astra", service_tiers=(["ultrafast"],) + id="gpt-6", + mode="chat", + max_input_tokens=9, + upstream_model="openai/gpt-6-astra", + service_tiers=(["ultrafast"],), + supported_reasoning_efforts=router.get_model_group_info("gpt-6-astra").supported_reasoning_efforts, ) (entry,) = json.loads(codex_model_list_body(listing, (("gpt-6", "gpt-6"),), router))["models"] assert (entry["slug"], entry["display_name"], [tier["id"] for tier in entry["service_tiers"]]) == ( @@ -529,6 +616,48 @@ def test_model_list_body_takes_stock_metadata_off_the_deployment_the_key_team_ro assert [entry["slug"] for entry in served("team-2")] == ["coding-model", "coding"] +def test_model_list_body_offers_what_model_group_info_reports_for_each_listed_model(): + """Declared, unknown and non-reasoning groups through a real router, a hidden alias read like its + target, and the requesting Codex's version deciding which declared levels it is offered.""" + vllm = {"model": "hosted_vllm/qa-local-coder", "api_base": "https://vllm.example/v1"} + router = litellm.Router( + model_list=[ + { + "model_name": "declared", + "litellm_params": vllm, + "model_info": {"supports_reasoning": True, "reasoning_effort_levels": ["low", "high", "max"]}, + }, + {"model_name": "unknown", "litellm_params": vllm}, + {"model_name": "non-reasoning", "litellm_params": vllm, "model_info": {"supports_reasoning": False}}, + ], + model_group_alias={"hidden-declared": {"model": "declared", "hidden": True}}, + ) + names = ("declared", "unknown", "non-reasoning", "hidden-declared") + listing = tuple( + {"id": name, "object": "model", "created": 0, "owned_by": "openai", "mode": "chat"} for name in names + ) + entries = tuple((name, name) for name in names) + + def offered(client_version): + body = codex_model_list_body(listing, entries, router, client_version=client_version) + return { + entry["slug"]: ( + [level["effort"] for level in entry["supported_reasoning_levels"]], + entry.get("default_reasoning_level"), + ) + for entry in json.loads(body)["models"] + } + + assert router.get_model_group_info("declared").supported_reasoning_efforts == ("low", "high", "max") + assert offered("0.165.0") == { + "declared": (["low", "high", "max"], "high"), + "unknown": ([], None), + "non-reasoning": ([], None), + "hidden-declared": (["low", "high", "max"], "high"), + } + assert offered("0.137.0")["declared"] == (["low", "high"], "high") + + def test_catalog_rows_without_a_router_carry_only_the_listing(): listing = ({"id": "plain", "object": "model", "created": 0, "owned_by": "openai", "mode": "chat"},) diff --git a/tests/unit/proxy/proxy_server/test_routes_models.py b/tests/unit/proxy/proxy_server/test_routes_models.py index 7be30c61f0c..33c95faaf2d 100644 --- a/tests/unit/proxy/proxy_server/test_routes_models.py +++ b/tests/unit/proxy/proxy_server/test_routes_models.py @@ -17,7 +17,7 @@ import litellm from litellm.proxy import proxy_server from litellm.proxy import utils as proxy_utils from litellm.proxy.utils import create_model_info_response -from litellm.types.router import DeploymentModelListingInfo +from litellm.types.router import DeploymentModelListingInfo, ModelGroupInfo from .conftest import normalize # type: ignore[import-not-found] @@ -49,6 +49,7 @@ def patched_models(monkeypatch): router.get_routable_upstream_model = MagicMock(return_value="gpt-4") router.get_configured_display_name = MagicMock(return_value=None) router.get_configured_service_tiers = MagicMock(return_value=()) + router.get_model_group_info = MagicMock(return_value=None) monkeypatch.setattr(proxy_server, "llm_router", router) monkeypatch.setattr(proxy_server, "prisma_client", MagicMock()) @@ -460,6 +461,28 @@ def test_codex_format_carries_configured_service_tiers(client, auth_as, patched_ assert all("service_tiers" not in m for m in openai_response.json()["data"]) +@pytest.mark.parametrize("params", [{}, {"scope": "expand"}]) +def test_codex_reasoning_levels_are_the_ones_the_requesting_codex_parses(client, auth_as, patched_models, params): + """Both listing paths hand the route's ``client_version`` to the catalog: Codex before 0.138.0 cannot + parse max, so it is offered there only to a newer release.""" + patched_models.get_model_group_info = MagicMock( + return_value=ModelGroupInfo( + model_group="gpt-4", providers=["openai"], supported_reasoning_efforts=("low", "max") + ) + ) + + with auth_as(): + current = client.get("/v1/models", params={**params, "client_version": "0.165.0"}) + older = client.get("/v1/models", params={**params, "client_version": "0.137.0"}) + + assert [[level["effort"] for level in m["supported_reasoning_levels"]] for m in current.json()["models"]] == [ + ["low", "max"] + ] * 2 + assert [[level["effort"] for level in m["supported_reasoning_levels"]] for m in older.json()["models"]] == [ + ["low"] + ] * 2 + + @pytest.mark.parametrize("params", [{}, {"scope": "expand"}]) def test_codex_service_tiers_are_read_for_the_key_team(client, auth_as, patched_models, params): """A tier and the upstream model that picks Codex's stock entry are read off the deployments the diff --git a/tests/unit/router_utils/test_reasoning_effort_capability.py b/tests/unit/router_utils/test_reasoning_effort_capability.py index d0564812c48..f36943f6d1a 100644 --- a/tests/unit/router_utils/test_reasoning_effort_capability.py +++ b/tests/unit/router_utils/test_reasoning_effort_capability.py @@ -8,7 +8,6 @@ import pytest import litellm from litellm._logging import JsonFormatter, verbose_logger from litellm.router_utils.reasoning_effort_capability import ( - REASONING_EFFORT_STRENGTH_ORDER, RequestedReasoningEffort, deployment_is_catalog_mapped, intersect_supported_reasoning_efforts, @@ -17,6 +16,7 @@ from litellm.router_utils.reasoning_effort_capability import ( resolve_supported_reasoning_efforts, thinking_budget_effort, ) +from litellm.router_utils.reasoning_effort_strength import REASONING_EFFORT_STRENGTH_ORDER class TestDeploymentIsCatalogMapped: diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 45f9630983f..140796cf5ec 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -10516,8 +10516,10 @@ export interface paths { * - client_version: Sent by Codex CLI (`?client_version=0.159.3`) when it fetches a * provider's model catalog. When present, the response is Codex's own * catalog shape (`{"models": [...]}`) built from the same listing, with - * each model's `model_info.service_tiers` as its service tiers; absent, - * the OpenAI shape below is returned + * each model's `model_info.service_tiers` as its service tiers and, for a + * model that Codex release does not know, the reasoning efforts + * `/model_group/info` reports that it can parse; absent, the OpenAI shape + * below is returned * - include_metadata: Include additional metadata in the response with fallback information * - fallback_type: Type of fallbacks to include ("general", "context_window", "content_policy") * Defaults to "general" when include_metadata=true @@ -21428,8 +21430,10 @@ export interface paths { * - client_version: Sent by Codex CLI (`?client_version=0.159.3`) when it fetches a * provider's model catalog. When present, the response is Codex's own * catalog shape (`{"models": [...]}`) built from the same listing, with - * each model's `model_info.service_tiers` as its service tiers; absent, - * the OpenAI shape below is returned + * each model's `model_info.service_tiers` as its service tiers and, for a + * model that Codex release does not know, the reasoning efforts + * `/model_group/info` reports that it can parse; absent, the OpenAI shape + * below is returned * - include_metadata: Include additional metadata in the response with fallback information * - fallback_type: Type of fallbacks to include ("general", "context_window", "content_policy") * Defaults to "general" when include_metadata=true