scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-15 09:12:18 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 311

fix(proxy): keep Codex tiers for unpriced deployments and match stock entries by declared provider

Browse source 
A deployment with no token price no longer votes a Codex service tier off,
an undescribed model keeps Codex's default reasoning summary parameter,
a Bedrock deployment naming its provider in custom_llm_provider gets Codex's
stock entry, and Bedrock provider namespaces follow the AWS model id format.
This commit is contained in:
ryan-crabbe-berri 2026-11-03 00:15:46 -08:00
parent 1526df0851
commit d5922c4bae
13 changed files with 291 additions and 116 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
litellm/llms/bedrock/common_utils.py
View file

@ -13,7 +13,7 @@ import warnings
from collections.abc import Callable, Iterator, Mapping, Sequence
from dataclasses import dataclass
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, get_args
from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias
from typing_extensions import (
ReadOnly,
@ -32,7 +32,6 @@ from pydantic import ConfigDict, TypeAdapter, ValidationError
import litellm
from litellm import verbose_logger
from litellm.constants import BEDROCK_INVOKE_PROVIDERS_LITERAL
from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
from litellm.litellm_core_utils.optional_imports import ensure_optional_import
from litellm.litellm_core_utils.prompt_templates.common_utils import schema_property_count
@ -1491,12 +1490,17 @@ def get_bedrock_base_model(model: str) -> str:
return model
def bedrock_vendor_model_name(model: str) -> str:
"""The name the model's vendor publishes it under, without Bedrock's routing, region and vendor
prefixes: ``bedrock/us.openai.gpt-6.1-sol`` and ``us-east-1/openai.gpt-6.1-sol`` are ``gpt-6.1-sol``."""
def bedrock_model_name_without_provider(model: str) -> str:
"""The model id without Bedrock's routing and region prefixes and without its provider namespace.
A Bedrock foundation model id is ``<provider>.<model>`` with a one-word provider (``anthropic.``,
``deepseek.``, ``moonshotai.``), so a base model whose first dot-separated part is one word drops it:
``bedrock/us.openai.gpt-6.1-sol`` and ``us-east-1/openai.gpt-6.1-sol`` are ``gpt-6.1-sol``,
``us.deepseek.r1-v1:0`` is ``r1-v1:0``. Any other base model, such as ``gpt-5.5`` or an imported model's
ARN id, is returned as is."""
base: Final = get_bedrock_base_model(model)
vendor, separator, name = base.partition(".")
return name if separator and vendor in get_args(BEDROCK_INVOKE_PROVIDERS_LITERAL) else base
provider, separator, name = base.partition(".")
return name if separator and provider.isalnum() else base
def bedrock_converse_accepts_redacted_thinking(model: str, base_model: str | None) -> bool:

60
litellm/proxy/common_utils/codex_model_catalog.py
View file

@ -6,8 +6,8 @@ feature on) and decodes the body as its own ``ModelsResponse``, ``{"models": [Mo
never the OpenAI list shape. Codex then ignores its own bundled catalog, so this body is all it knows
about the provider's models. It is built from the rows ``/v1/models`` already lists: a model Codex
knows, by its id or by the upstream model its deployments call once the provider's own prefixes are
stripped, keeps Codex's own entry for it, minus any service tier some deployment behind it has no
price for; any other model gets an entry translated from the capabilities its deployments' model info
stripped, keeps Codex's own entry for it, minus any service tier some deployment behind it prices tokens
but not that tier for; any other model gets an entry translated from the capabilities its deployments' model info
declares (see ``codex_model_translation``). ``model_info.service_tiers`` becomes the entry's
``service_tiers``, which Codex turns into slash commands (``/ultrafast``) that send ``service_tier``
upstream. A ``model_info.max_input_tokens`` the operator sets replaces a known model's
@ -35,16 +35,18 @@ from typing import TYPE_CHECKING, Final, Literal, TypeAlias
from pydantic import ConfigDict, TypeAdapter, ValidationError
from litellm._logging import verbose_proxy_logger
from litellm.llms.bedrock.common_utils import bedrock_vendor_model_name
from litellm.litellm_core_utils.get_llm_provider_logic import declared_provider
from litellm.llms.bedrock.common_utils import bedrock_model_name_without_provider
from litellm.proxy.common_utils.codex_model_translation import (
TIER_PRICE_KEYS,
TOKEN_PRICE_KEY,
CodexDeploymentCapabilities,
CodexFallbackModel,
CodexServiceTier,
TierId,
codex_fallback_entry,
combined_capabilities,
every_deployment_prices,
no_priced_deployment_lacks_tier,
)
from litellm.types.llms.base_model import LiteLLMBaseModel
@ -122,7 +124,8 @@ CodexAliasDefaults: TypeAlias = Literal["codex", "deployment"]
@dataclass(frozen=True, slots=True)
class CodexCatalogRow:
"""What the catalog needs to know about one listed model: its public id, the listing's mode and
input limit, the upstream model behind it, the reasoning effort its deployments pin, and the
input limit, the upstream model behind it and the provider its deployment declares, the reasoning
effort its deployments pin, and the
`model_info` values the operator set; `service_tiers` holds one raw `model_info.service_tiers`
value per deployment behind the id, None where a deployment sets none, and
`configured_max_input_tokens` one `model_info.max_input_tokens` per deployment the same way.
@ -133,6 +136,7 @@ class CodexCatalogRow:
mode: str | None = None
max_input_tokens: int | None = None
upstream_model: str | None = None
upstream_provider: str | None = None
reasoning_effort: str | None = None
display_name: str | None = None
service_tiers: tuple[object, ...] = ()
@ -242,36 +246,44 @@ def _codex_can_drive(model_id: str, mode: str | None) -> bool:
return "*" not in model_id and (mode is None or mode in CODEX_CHAT_MODES)
_VENDOR_MODEL_NAMES: Final[Mapping[str, Callable[[str], str]]] = MappingProxyType(
{"bedrock": bedrock_vendor_model_name, "bedrock_mantle": bedrock_vendor_model_name}
_PROVIDER_MODEL_NAMES: Final[Mapping[str, Callable[[str], str]]] = MappingProxyType(
{"bedrock": bedrock_model_name_without_provider, "bedrock_mantle": bedrock_model_name_without_provider}
)
def _vendor_model_name(upstream_model: str) -> str:
provider, _, model = upstream_model.partition("/")
name_for: Final = _VENDOR_MODEL_NAMES.get(provider)
def _provider_model_name(upstream_model: str, upstream_provider: str | None) -> str:
"""The upstream model without the prefixes its provider adds, the provider being the one the deployment
declares in `custom_llm_provider`, else the model's `provider/` prefix."""
provider: Final = declared_provider(upstream_model, upstream_provider)
if provider is None:
return upstream_model
model: Final = upstream_model.removeprefix(f"{provider}/")
name_for: Final = _PROVIDER_MODEL_NAMES.get(provider)
return model if name_for is None else name_for(model)
def _stock_for(row: CodexCatalogRow, stock: Mapping[str, CodexStockModel]) -> CodexStockModel | None:
"""Codex's entry for the row's id, else for its upstream model as written, else for that model's name
once the provider's own prefixes are stripped (`bedrock/us.openai.gpt-6.1-sol` is `gpt-6.1-sol`)."""
once its provider's own prefixes are stripped (`bedrock/us.openai.gpt-6.1-sol` is `gpt-6.1-sol`)."""
names: Final = (
(row.id,)
if row.upstream_model is None
else (row.id, row.upstream_model, _vendor_model_name(row.upstream_model))
else (row.id, row.upstream_model, _provider_model_name(row.upstream_model, row.upstream_provider))
)
return next((stock[name] for name in names if name in stock), None)
def _priced_tiers(row: CodexCatalogRow, stock_model: CodexStockModel) -> Mapping[str, object]:
"""Codex's own tiers for the model without the ones some deployment behind the row has no price for,
or nothing when every deployment prices them all. Sending a tier the upstream does not sell fails the
request, and the cost map prices a tier for exactly the models that sell it."""
"""Codex's own tiers for the model without the ones some deployment behind the row prices tokens but not
that tier for, or nothing when none is dropped. Sending a tier the upstream does not sell fails the
request, and the cost map prices a tier for exactly the models that sell it, while a deployment it has no
token price for at all says nothing either way."""
deployments: Final = row.deployment_capabilities
tiers: Final = tuple(tier for tier in stock_model.service_tiers if every_deployment_prices(tier.id, deployments))
tiers: Final = tuple(
tier for tier in stock_model.service_tiers if no_priced_deployment_lacks_tier(tier.id, deployments)
)
speed_tiers: Final = tuple(
tier for tier in stock_model.additional_speed_tiers if every_deployment_prices(tier, deployments)
tier for tier in stock_model.additional_speed_tiers if no_priced_deployment_lacks_tier(tier, deployments)
)
if tiers == stock_model.service_tiers and speed_tiers == stock_model.additional_speed_tiers:
return _NO_UPDATE
@ -457,8 +469,8 @@ def codex_models_response_json(
`max_context_window`. A row listed under another name than the Codex model its `upstream_model` names (an alias) takes that model's
entry; `alias_defaults="deployment"` drops the defaults Codex picks for that model in favor of
what the alias's deployments pin, while `"codex"` keeps them, as Codex would use them for the
model's own name. Codex's own service tiers stay on a known model's entry only when every deployment
behind the row prices them, and only while the operator configures none. `client_version` is the
model's own name. Codex's own service tiers stay on a known model's entry only when no deployment
behind the row prices tokens without pricing them, and only while the operator configures none. `client_version` is the
requesting Codex's version, which decides the reasoning levels a fallback entry may offer; None counts
as a version too old for custom levels.
"""
@ -492,14 +504,17 @@ def deployment_capabilities(model_info: Mapping[str, object] | None, model_id: s
Model info that does not validate declares nothing, logged, so one bad value never fails the listing."""
if model_info is None:
return CodexDeploymentCapabilities()
prices_tokens: Final = model_info.get(TOKEN_PRICE_KEY) is not None
priced: Final = frozenset(key for key in TIER_PRICE_KEYS if model_info.get(key) is not None)
try:
return CodexDeploymentCapabilities.model_validate({**model_info, "priced_service_tier_keys": priced})
return CodexDeploymentCapabilities.model_validate(
{**model_info, "prices_tokens": prices_tokens, "priced_service_tier_keys": priced}
)
except ValidationError as e:
verbose_proxy_logger.warning(
"model info of a deployment of %s declares no capability to Codex: %s", model_id, e.errors()[0]["msg"]
)
return CodexDeploymentCapabilities(priced_service_tier_keys=priced)
return CodexDeploymentCapabilities(prices_tokens=prices_tokens, priced_service_tier_keys=priced)
def _deployment_capabilities(
@ -522,6 +537,9 @@ def _catalog_row(
mode=row.get("mode"),
max_input_tokens=row.get("max_input_tokens"),
upstream_model=registry.resolve(lookup_id, kind="upstream", team_id=team_id) if registry is not None else None,
upstream_provider=(
registry.resolve(lookup_id, kind="upstream_provider", team_id=team_id) if registry is not None else None
),
display_name=registry.group(label_id, field="display_name") if registry is not None else None,
service_tiers=registry.group(lookup_id, field="service_tiers", team_id=team_id) if registry is not None else (),
supported_reasoning_efforts=_group_reasoning_efforts(row, lookup_id, registry, team_id),

42
litellm/proxy/common_utils/codex_model_translation.py
View file

@ -26,14 +26,18 @@ CodexInputModality: TypeAlias = Literal["text", "image"]
_TEXT_ONLY: Final[tuple[CodexInputModality, ...]] = ("text",)
_TEXT_AND_IMAGE: Final[tuple[CodexInputModality, ...]] = ("text", "image")
_TIER_PRICE_BASE_KEY: Final = "input_cost_per_token"
TIER_PRICE_KEYS: Final = tuple(f"{_TIER_PRICE_BASE_KEY}{suffix}" for suffix in SERVICE_TIER_COST_KEY_SUFFIXES)
TOKEN_PRICE_KEY: Final = "input_cost_per_token"
TIER_PRICE_KEYS: Final = tuple(f"{TOKEN_PRICE_KEY}{suffix}" for suffix in SERVICE_TIER_COST_KEY_SUFFIXES)
_FIRST_CODEX_WITH_CUSTOM_EFFORTS: Final = Version("0.138.0")
_EFFORTS_EVERY_CODEX_PARSES: Final = frozenset({"none", "minimal", "low", "medium", "high", "xhigh"})
_PREFERRED_DEFAULT_EFFORT: Final = "medium"
_CODEX_UNSET_EFFORT: Final = "none"
def _unset(value: object) -> bool:
return value is None
class CodexServiceTier(LiteLLMBaseModel):
"""A `ModelServiceTier` as Codex reads it: the slash command is `name` lowercased, and toggling
it sends `id` as the request's `service_tier`."""
@ -75,7 +79,7 @@ class CodexFallbackModel(LiteLLMBaseModel):
slug: str
display_name: str
description: NoneType = None
default_reasoning_level: str | None = Field(default=None, exclude_if=lambda level: level is None)
default_reasoning_level: str | None = Field(default=None, exclude_if=_unset)
supported_reasoning_levels: tuple[CodexFallbackReasoningLevel, ...] = ()
shell_type: Literal["unified_exec"] = "unified_exec"
visibility: Literal["list"] = "list"
@ -85,7 +89,7 @@ class CodexFallbackModel(LiteLLMBaseModel):
availability_nux: NoneType = None
upgrade: NoneType = None
support_verbosity: Literal[False] = False
supports_reasoning_summary_parameter: bool = False
supports_reasoning_summary_parameter: bool | None = Field(default=None, exclude_if=_unset)
supports_reasoning_summaries: bool = False
supports_parallel_tool_calls: bool = False
default_verbosity: NoneType = None
@ -101,7 +105,8 @@ class CodexFallbackModel(LiteLLMBaseModel):
class CodexDeploymentCapabilities(LiteLLMBaseModel):
"""What one deployment's model info says about the capabilities a Codex entry describes, None where
it says nothing, and the input price keys of the service tiers it has a price for."""
it says nothing, whether it has a token price at all, and the input price keys of the service tiers it
has a price for."""
model_config = ConfigDict(frozen=True, extra="ignore")
@ -110,14 +115,19 @@ class CodexDeploymentCapabilities(LiteLLMBaseModel):
supports_function_calling: bool | None = None
supports_parallel_function_calling: bool | None = None
supports_reasoning: bool | None = None
prices_tokens: bool = False
priced_service_tier_keys: frozenset[str] = frozenset()
def every_deployment_prices(tier_id: str, deployments: Sequence[CodexDeploymentCapabilities]) -> bool:
"""Whether every deployment has a price for `tier_id`, the way cost tracking prices a request sent at
that tier; a tier cost tracking has no price key for is priced by none."""
price_key: Final = get_service_tier_cost_key(_TIER_PRICE_BASE_KEY, tier_id)
return all(price_key in deployment.priced_service_tier_keys for deployment in deployments)
def no_priced_deployment_lacks_tier(tier_id: str, deployments: Sequence[CodexDeploymentCapabilities]) -> bool:
"""Whether every deployment that has a token price also has one for `tier_id`, the way cost tracking
prices a request sent at that tier. A deployment with no token price at all (a subscription upstream,
a deployment name the cost map does not know) says nothing about tiers, so it does not vote, and a tier
cost tracking has no price key for is lacked by every deployment that prices tokens."""
price_key: Final = get_service_tier_cost_key(TOKEN_PRICE_KEY, tier_id)
return all(
price_key in deployment.priced_service_tier_keys for deployment in deployments if deployment.prices_tokens
)
@dataclass(frozen=True, slots=True)
@ -205,12 +215,12 @@ def codex_fallback_entry(
) -> CodexFallbackModel:
"""Codex's entry for a model it does not know, from the capabilities its deployments share.
A capability no deployment declares keeps Codex's own default for an unknown model, except the two
that send something a model may reject: the `reasoning.summary` parameter goes out only to a model
that reasons, and the freeform `apply_patch` tool only to one that calls tools.
A capability no deployment declares keeps Codex's own default for an unknown model, except the freeform
`apply_patch` tool, which goes out only to a model that calls tools: `supports_reasoning_summary_parameter`
is left out, which Codex reads as true, and the fields only Codex before 0.150.0 reads keep the false
it requires.
"""
levels, default_level = codex_reasoning_levels(reasoning_efforts, client_version)
reasons: Final = capabilities.supports_reasoning is True
return CodexFallbackModel(
slug=slug,
display_name=display_name,
@ -218,8 +228,8 @@ def codex_fallback_entry(
supported_reasoning_levels=levels,
priority=priority,
service_tiers=service_tiers,
supports_reasoning_summary_parameter=reasons,
supports_reasoning_summaries=reasons,
supports_reasoning_summary_parameter=capabilities.supports_reasoning,
supports_reasoning_summaries=capabilities.supports_reasoning is True,
supports_parallel_tool_calls=capabilities.supports_parallel_function_calling is True,
apply_patch_tool_type="freeform" if capabilities.supports_function_calling is True else None,
input_modalities=_TEXT_ONLY if capabilities.supports_vision is False else _TEXT_AND_IMAGE,

24
litellm/router_api.py
View file

@ -143,6 +143,7 @@ ResolveKind: TypeAlias = Literal[
"team_model",
"recognized",
"upstream",
"upstream_provider",
"group_of_id",
"wildcard_access_group",
"team_model_alternatives",
@ -685,7 +686,7 @@ class RouterRegistry:
self,
name: str,
*,
kind: Literal["alias", "upstream"],
kind: Literal["alias", "upstream", "upstream_provider"],
team_id: str | None = None,
custom_llm_provider: str | None = None,
served_model: str | None = None,
@ -765,6 +766,8 @@ class RouterRegistry:
- ``team_model``: a team's public name to its group (was ``map_team_model``; ``team_id`` required)
- ``recognized``: whether the router knows the name (was ``is_recognized_model``)
- ``upstream``: the upstream model a name routes to (was ``get_routable_upstream_model``)
- ``upstream_provider``: the provider the deployment ``upstream`` reads declares, by its
``custom_llm_provider`` or its model's ``provider/`` prefix
- ``group_of_id``: the group of a deployment id (was ``get_model_group``)
- ``wildcard_access_group``: whether an access group grants a wildcard route
(was ``is_model_access_group_for_wildcard_route``)
@ -794,10 +797,8 @@ class RouterRegistry:
return name if served_model is None else r._core_resolve_served_model_name(name, served_model)
if kind == "recognized":
return r._core_is_recognized_model(name)
if kind == "upstream":
return r._core_get_routable_upstream_model(name, team_id=team_id)
if kind == "search_tool_support":
return r._core_get_search_tool_support(name, team_id=team_id)
if kind in ("upstream", "upstream_provider", "search_tool_support"):
return self._resolve_routable(name, kind, team_id)
if kind == "group_of_id":
return r._core_get_model_group(id=name)
if kind == "wildcard_access_group":
@ -806,6 +807,19 @@ class RouterRegistry:
return r._core_team_model_has_alternatives(name)
raise ValueError(f"unknown resolve kind {kind!r}")
def _resolve_routable(
self,
name: str,
kind: Literal["upstream", "upstream_provider", "search_tool_support"],
team_id: str | None,
) -> str | tuple[bool | None, ...] | None:
r: Final = self._router
if kind == "upstream":
return r._core_get_routable_upstream_model(name, team_id=team_id)
if kind == "upstream_provider":
return r._core_get_routable_upstream_provider(name, team_id=team_id)
return r._core_get_search_tool_support(name, team_id=team_id)
def aliases(
self,
) -> "dict[str, str | RouterModelGroupAliasItem]": # mutable-ok: the model_group_alias attribute, the live dict

41
litellm/router_python_core.py
View file

@ -137,6 +137,7 @@ from litellm.litellm_core_utils.deployment_pricing import deployment_pricing_mod
from litellm.litellm_core_utils.get_litellm_params import SUPPLEMENTAL_KWARGS_KEYS
from litellm.litellm_core_utils.get_llm_provider_logic import (
declared_authenticating_provider,
declared_provider,
inferred_provider,
is_registered_custom_provider,
resolve_model_and_provider_for_metadata,
@ -786,6 +787,16 @@ def _operator_max_input_tokens(deployment: Mapping[str, object]) -> int | None:
return coerce_token_limit(model_info.get("max_input_tokens"))
def _deployment_params(deployment: DeploymentTypedDict) -> Mapping[str, object]:
return deployment["litellm_params"]
def _model_info_key(deployment: DeploymentTypedDict) -> tuple[str, str] | None:
deployment_id: Final = _configured_model_info(deployment).get("id")
model: Final = _deployment_params(deployment).get("model")
return (deployment_id, model) if isinstance(deployment_id, str) and isinstance(model, str) else None
def _search_tool_support(deployment: DeploymentTypedDict) -> bool | None:
from litellm.responses.litellm_completion_transformation.tool_search import chat_config_builds_anthropic_request
@ -12753,15 +12764,10 @@ class PythonCore(DeprecatedRouterMembers):
Returns an empty tuple for wildcard-expanded or unknown names.
"""
return tuple(
self._core_cached_deployment_model_info(deployment_id, model)
if isinstance(deployment_id := _configured_model_info(deployment).get("id"), str)
and isinstance(
model := _LITELLM_PARAMS_ADAPTER.validate_python(deployment["litellm_params"]).get("model"), str
)
else None
for deployment in self._routable_deployments(model_name, team_id)
keys: Final = tuple(
_model_info_key(deployment) for deployment in self._routable_deployments(model_name, team_id)
)
return tuple(None if key is None else self._core_cached_deployment_model_info(*key) for key in keys)
def _core_get_routable_upstream_model(self, model_name: str, team_id: str | None = None) -> str | None:
"""
@ -12774,6 +12780,25 @@ class PythonCore(DeprecatedRouterMembers):
deployment: Final = next(iter(self._routable_deployments(model_name, team_id)), None)
return deployment["litellm_params"].get("model") if deployment is not None else None
def _core_get_routable_upstream_provider(self, model_name: str, team_id: str | None = None) -> str | None:
"""
Return the provider the deployment _core_get_routable_upstream_model
reads declares: its `litellm_params.custom_llm_provider`, else the
`provider/` prefix of its `litellm_params.model`.
Returns None when that deployment declares neither, and wherever
_core_get_routable_upstream_model returns None.
"""
deployment: Final = next(iter(self._routable_deployments(model_name, team_id)), None)
if deployment is None:
return None
params: Final = _deployment_params(deployment)
model: Final = params.get("model")
provider: Final = params.get("custom_llm_provider")
return declared_provider(
model if isinstance(model, str) else None, provider if isinstance(provider, str) else None
)
def _core_get_credential_deployment(self, model_id: str, team_id: str | None = None) -> Deployment | None:
"""
The deployment a passthrough endpoint (files, batches, etc.) resolves for a

11
tests/unit/llms/bedrock/test_bedrock_common_utils.py
View file

@ -9,7 +9,7 @@ from litellm.llms.bedrock.common_utils import(
NativeOutputFallback,
bedrock_native_output_is_opt_in,
bedrock_strict_tools_fallback,
bedrock_vendor_model_name,
bedrock_model_name_without_provider,
deployment_model_info,
extract_model_name_from_bedrock_arn,
get_bedrock_base_model,
@ -312,12 +312,15 @@ def test_legacy_mantle_route_prefix_stripped_for_cost_lookup():
("us-east-1/openai.gpt-6.1-sol", "gpt-6.1-sol"),
("openai.gpt-5.5", "gpt-5.5"),
("arn:aws:bedrock:us-east-1:123456789012:inference-profile/us.meta.llama4-scout-17b", "llama4-scout-17b"),
("us.deepseek.r1-v1:0", "r1-v1:0"),
("moonshotai.kimi-k2-thinking", "kimi-k2-thinking"),
("bedrock/qwen.qwen3-32b-v1:0", "qwen3-32b-v1:0"),
("gpt-5.5", "gpt-5.5"),
("unknownvendor.some-model", "unknownvendor.some-model"),
("arn:aws:bedrock:us-east-1:123456789012:imported-model/abc123", "abc123"),
],
)
def test_bedrock_vendor_model_name_strips_routing_region_and_vendor_prefixes_only(model, expected):
assert bedrock_vendor_model_name(model) == expected
def test_bedrock_model_name_without_provider_strips_routing_region_and_provider_prefixes_only(model, expected):
assert bedrock_model_name_without_provider(model) == expected
def test_output_config_effort_normalization_uses_model_info_ceiling(monkeypatch):

1
tests/unit/proxy/client/cli/test_agents.py
View file

@ -1041,6 +1041,7 @@ class TestCodexModelSync:
assert entry["experimental_supported_tools"] == []
assert entry["support_verbosity"] is False
assert entry["supports_reasoning_summaries"] is False
assert "supports_reasoning_summary_parameter" not in entry
assert entry["supports_parallel_tool_calls"] is False
for nullable in ("description", "availability_nux", "upgrade", "default_verbosity", "apply_patch_tool_type"):
assert nullable in entry and entry[nullable] is None

125
tests/unit/proxy/common_utils/test_codex_model_catalog.py
View file

@ -87,29 +87,10 @@ def test_body_is_exactly_codex_models_response():
assert [entry["slug"] for entry in body["models"]] == ["gpt-6-astra"]
def test_unknown_model_entry_carries_every_field_codex_deserializes_without_a_default():
"""Codex 0.166.0 `codex-rs/protocol/src/openai_models.rs` ModelInfo, the fields with no
`#[serde(default)]`, read on 2026-11-02; `base_instructions` because its `ModelsResponse`
decoder rejects an entry with neither it nor `model_messages.instructions_template`."""
def test_unknown_model_entry_takes_its_slug_window_and_prompt_from_the_row():
"""The fields every Codex release decodes are pinned in test_codex_model_translation.py."""
(entry,) = _models(_row("gpt-6-astra"))
assert entry.keys() >= {
"slug",
"display_name",
"description",
"supported_reasoning_levels",
"shell_type",
"visibility",
"supported_in_api",
"priority",
"availability_nux",
"upgrade",
"support_verbosity",
"default_verbosity",
"apply_patch_tool_type",
"truncation_policy",
"experimental_supported_tools",
}
assert (entry["slug"], entry["display_name"], entry["visibility"], entry["supported_in_api"]) == (
"gpt-6-astra",
"gpt-6-astra",
@ -722,6 +703,7 @@ def test_catalog_rows_read_the_router_by_each_entry_lookup_id():
mode="chat",
max_input_tokens=7,
upstream_model="openai/gpt-5.5",
upstream_provider="openai",
display_name="Team 5.5",
service_tiers=(["ultrafast"], None),
supported_reasoning_efforts=router.get_model_group_info(
@ -733,6 +715,7 @@ def test_catalog_rows_read_the_router_by_each_entry_lookup_id():
CodexCatalogRow(
id="plain",
upstream_model="openai/some-unmapped-model",
upstream_provider="openai",
service_tiers=(None,),
search_tool_support=(None,),
configured_max_input_tokens=(None,),
@ -766,6 +749,7 @@ def test_catalog_rows_read_an_alias_off_its_target_under_the_alias_name():
mode="chat",
max_input_tokens=9,
upstream_model="openai/gpt-6-astra",
upstream_provider="openai",
service_tiers=(["ultrafast"],),
supported_reasoning_efforts=router.get_model_group_info("gpt-6-astra").supported_reasoning_efforts,
search_tool_support=(None,),
@ -974,6 +958,41 @@ def test_a_model_codex_knows_keeps_codexs_entry_under_any_provider_prefix(upstre
assert entry["supported_reasoning_levels"] == stock["supported_reasoning_levels"]
@pytest.mark.parametrize("model", ["us.openai.{slug}", "converse/global.openai.{slug}"])
def test_a_bedrock_deployment_naming_its_provider_apart_from_the_model_keeps_codexs_entry(model: str) -> None:
router: Final = litellm.Router(
model_list=[
{
"model_name": "team-model",
"litellm_params": {
"model": model.format(slug=_SOL_SLUG),
"custom_llm_provider": "bedrock",
"aws_region_name": "us-east-1",
},
}
]
)
entry: Final = _served(router, "team-model")["team-model"]
assert entry["model_messages"] == bundled_codex_models()[_SOL_SLUG].model_dump(mode="json")["model_messages"]
def test_an_upstream_with_no_provider_prefix_matches_only_by_its_own_name() -> None:
(unknown, known) = json.loads(
codex_models_response_json(
(
_row("first", upstream_model=f"us.openai.{_SOL_SLUG}"),
_row("second", upstream_model=_SOL_SLUG),
),
instructions=_PROMPT,
).json
)["models"]
assert "model_messages" not in unknown and unknown["base_instructions"] == _PROMPT
assert known["model_messages"] == bundled_codex_models()[_SOL_SLUG].model_dump(mode="json")["model_messages"]
@pytest.mark.parametrize(
"upstream",
["bedrock/us.openai.{slug}-mini", "bedrock/us.meta.{slug}x", "bedrock_mantle/openai.{slug}.1", "{slug}-0"],
@ -1002,7 +1021,9 @@ def _listing(*names: str, max_input_tokens: int | None = None) -> tuple[dict[str
)
def _served(router: litellm.Router, *names: str, team_id: str | None = None, **listing: int) -> dict[str, dict]:
def _served(
router: litellm.Router, *names: str, team_id: str | None = None, **listing: int
) -> dict[str, Mapping[str, object]]:
body: Final = codex_model_list_body(
_listing(*names, **listing), tuple((name, name) for name in names), router, team_id, client_version="0.166.0"
)
@ -1077,20 +1098,20 @@ def test_a_claude_like_deployment_gets_codexs_coding_setup() -> None:
supports_reasoning=True,
),
_vllm("plain", "h12"),
_vllm("no-reasoning", "h12b", supports_reasoning=False),
]
)
served: Final = _served(router, "claude-like", "plain")
served: Final = _served(router, "claude-like", "plain", "no-reasoning")
claude_like: Final = served["claude-like"]
assert claude_like["apply_patch_tool_type"] == "freeform"
assert claude_like["truncation_policy"] == {"mode": "tokens", "limit": 10000}
assert claude_like["supports_reasoning_summary_parameter"] is True
assert claude_like["context_window"] == 1_000_000
assert (served["plain"]["apply_patch_tool_type"], served["plain"]["supports_reasoning_summary_parameter"]) == (
None,
False,
)
assert served["plain"]["apply_patch_tool_type"] is None
assert "supports_reasoning_summary_parameter" not in served["plain"]
assert served["no-reasoning"]["supports_reasoning_summary_parameter"] is False
def test_capabilities_are_read_only_off_the_deployments_the_key_team_routes_to() -> None:
@ -1112,7 +1133,8 @@ def _tier_fields(entry: Mapping[str, object]) -> tuple[object, object, object]:
def test_codexs_own_tiers_stay_only_when_every_deployment_behind_the_name_prices_them() -> None:
stock: Final = bundled_codex_models()[_SOL_SLUG].model_dump(mode="json")
assert stock["service_tiers"] and stock["additional_speed_tiers"] and stock["default_service_tier"]
priority_price: Final = {"input_cost_per_token_priority": 1e-05}
token_price: Final = {"input_cost_per_token": 1e-06}
priority_price: Final = {**token_price, "input_cost_per_token_priority": 1e-05}
def upstream(host: str) -> dict[str, str]:
return {"model": f"hosted_vllm/{_SOL_SLUG}", "api_base": f"https://{host}.example/v1"}
@ -1122,7 +1144,9 @@ def test_codexs_own_tiers_stay_only_when_every_deployment_behind_the_name_prices
{"model_name": "all-priced", "litellm_params": upstream("a"), "model_info": priority_price},
{"model_name": "all-priced", "litellm_params": upstream("b"), "model_info": priority_price},
{"model_name": "half-priced", "litellm_params": upstream("c"), "model_info": priority_price},
{"model_name": "half-priced", "litellm_params": upstream("d")},
{"model_name": "half-priced", "litellm_params": upstream("d"), "model_info": token_price},
{"model_name": "priced-and-unknown", "litellm_params": upstream("f"), "model_info": priority_price},
{"model_name": "priced-and-unknown", "litellm_params": {"model": "azure/prod-sol-eastus"}},
{
"model_name": "configured",
"litellm_params": upstream("e"),
@ -1131,14 +1155,55 @@ def test_codexs_own_tiers_stay_only_when_every_deployment_behind_the_name_prices
]
)
served: Final = _served(router, "all-priced", "half-priced", "configured")
served: Final = _served(router, "all-priced", "half-priced", "priced-and-unknown", "configured")
assert _tier_fields(served["all-priced"]) == _tier_fields(stock)
assert _tier_fields(served["priced-and-unknown"]) == _tier_fields(stock)
assert _tier_fields(served["half-priced"]) == ([], [], None)
assert served["half-priced"]["model_messages"] == stock["model_messages"]
assert [tier["id"] for tier in served["configured"]["service_tiers"]] == ["priority"]
def _tiered_stock(default_service_tier: str) -> Mapping[str, CodexStockModel]:
tiers: Final = (
CodexServiceTier(id="priority", name="Fast", description="faster"),
CodexServiceTier(id="flex", name="Flex", description="cheaper"),
)
return MappingProxyType(
{
"tiered": CodexStockModel(
slug="tiered",
display_name="Tiered",
priority=0,
visibility="list",
service_tiers=tiers,
default_service_tier=default_service_tier,
)
}
)
@pytest.mark.parametrize(("default_tier", "kept_default"), [("priority", "priority"), ("flex", None)])
def test_dropping_one_stock_tier_keeps_the_default_only_while_it_is_still_offered(default_tier, kept_default) -> None:
priority_and_flex: Final = CodexDeploymentCapabilities(
prices_tokens=True,
priced_service_tier_keys=frozenset(
get_service_tier_cost_key("input_cost_per_token", tier) for tier in ("priority", "flex")
),
)
priority_only: Final = CodexDeploymentCapabilities(
prices_tokens=True,
priced_service_tier_keys=frozenset({get_service_tier_cost_key("input_cost_per_token", "priority")}),
)
row: Final = _row("tiered", deployment_capabilities=(priority_and_flex, priority_only))
(entry,) = json.loads(
codex_models_response_json((row,), stock=_tiered_stock(default_tier), instructions=_PROMPT).json
)["models"]
assert [tier["id"] for tier in entry["service_tiers"]] == ["priority"]
assert entry["default_service_tier"] == kept_default
def test_model_info_that_does_not_validate_declares_no_capability_but_keeps_its_prices() -> None:
priced: Final = get_service_tier_cost_key("input_cost_per_token", "priority")

56
tests/unit/proxy/common_utils/test_codex_model_translation.py
View file

@ -1,5 +1,4 @@
import json
from itertools import product
from typing import Final
import pytest
@ -11,11 +10,12 @@ from litellm.proxy.common_utils.codex_model_translation import (
CodexServiceTier,
codex_fallback_entry,
combined_capabilities,
every_deployment_prices,
no_priced_deployment_lacks_tier,
)
_PROMPT: Final = "base prompt"
_SILENT: Final = CodexDeploymentCapabilities()
_ABSENT: Final = object()
def _entry(capabilities: CodexCapabilities, **kwargs: object) -> dict[str, object]:
@ -42,21 +42,21 @@ def test_a_model_that_calls_tools_and_reasons_gets_codexs_own_coding_setup():
@pytest.mark.parametrize(
("flag", "field", "when_true", "otherwise"),
("flag", "field", "by_value"),
[
("supports_function_calling", "apply_patch_tool_type", "freeform", None),
("supports_reasoning", "supports_reasoning_summary_parameter", True, False),
("supports_reasoning", "supports_reasoning_summaries", True, False),
("supports_parallel_function_calling", "supports_parallel_tool_calls", True, False),
("supports_function_calling", "apply_patch_tool_type", {True: "freeform", False: None, None: None}),
("supports_reasoning", "supports_reasoning_summary_parameter", {True: True, False: False, None: _ABSENT}),
("supports_reasoning", "supports_reasoning_summaries", {True: True, False: False, None: False}),
("supports_parallel_function_calling", "supports_parallel_tool_calls", {True: True, False: False, None: False}),
],
)
@pytest.mark.parametrize("value", [True, False, None])
def test_a_feature_that_sends_something_upstream_is_on_only_when_every_deployment_declares_it(
flag, field, when_true, otherwise, value
):
def test_each_codex_feature_follows_what_the_deployments_declare(flag, field, by_value, value):
"""A missing `supports_reasoning_summary_parameter` is Codex's own default (true, `default_true` in
`codex-rs/protocol/src/openai_models.rs` at rust-v0.166.0), so a model nobody describes keeps it."""
entry: Final = _entry(CodexCapabilities(**{flag: value}))
assert entry[field] == (when_true if value is True else otherwise)
assert entry.get(field, _ABSENT) == by_value[value]
@pytest.mark.parametrize(
@ -124,23 +124,22 @@ _CODEX_MODALITIES: Final = {"text", "image"}
_CODEX_TRUNCATION_MODES: Final = {"bytes", "tokens"}
@pytest.mark.parametrize(
"flags", list(product((True, False, None), repeat=4)), ids=lambda flags: "-".join(map(str, flags))
)
def test_every_entry_carries_the_fields_and_enum_values_every_supported_codex_decodes(flags):
@pytest.mark.parametrize("described", [True, False], ids=["described", "undescribed"])
def test_every_entry_carries_the_fields_and_enum_values_every_supported_codex_decodes(described):
"""`codex-rs/protocol/src/openai_models.rs` ModelInfo at rust-v0.140.0 and rust-v0.166.0, the fields with
no `#[serde(default)]` and the variants of InputModality, ConfigShellToolType, ModelVisibility,
ApplyPatchToolType and TruncationMode in both, read on 2026-11-02. Codex rejects the whole catalog when
one entry does not decode."""
vision, tools, parallel, reasoning = flags
entry: Final = _entry(
CodexCapabilities(
context_window=128_000,
supports_vision=vision,
supports_function_calling=tools,
supports_parallel_function_calling=parallel,
supports_reasoning=reasoning,
supports_vision=False,
supports_function_calling=True,
supports_parallel_function_calling=True,
supports_reasoning=True,
)
if described
else CodexCapabilities()
)
assert entry.keys() >= _REQUIRED_SINCE_CODEX_0_140
@ -198,21 +197,28 @@ def test_the_window_is_the_smallest_any_deployment_declares(windows, listed, exp
def _priced(*tier_ids: str) -> CodexDeploymentCapabilities:
return CodexDeploymentCapabilities(
priced_service_tier_keys=frozenset(get_service_tier_cost_key("input_cost_per_token", tier) for tier in tier_ids)
prices_tokens=True,
priced_service_tier_keys=frozenset(
get_service_tier_cost_key("input_cost_per_token", tier) for tier in tier_ids
),
)
@pytest.mark.parametrize(
("deployments", "tier", "priced"),
("deployments", "tier", "kept"),
[
((_priced("priority"), _priced("priority", "flex")), "priority", True),
((_priced("priority"), _priced("flex")), "priority", False),
((_priced("priority"), _SILENT), "priority", False),
((_priced("priority"), _priced()), "priority", False),
((_priced("priority"), _SILENT), "priority", True),
((_SILENT, _SILENT), "priority", True),
((_priced("priority"),), "fast", True),
((_priced("flex"),), "fast", False),
((_priced("priority", "flex"),), "made-up-tier", False),
((), "priority", True),
],
)
def test_a_tier_is_priced_only_when_every_deployment_prices_it_the_way_cost_tracking_does(deployments, tier, priced):
assert every_deployment_prices(tier, deployments) is priced
def test_a_tier_stays_unless_a_deployment_that_prices_tokens_has_no_price_for_it(deployments, tier, kept):
"""A deployment with no token price at all (a subscription upstream, a deployment name the cost map does
not know) says nothing about which tiers its upstream sells, so it does not vote."""
assert no_priced_deployment_lacks_tier(tier, deployments) is kept

1
tests/unit/proxy/proxy_server/test_routes_models.py
View file

@ -56,6 +56,7 @@ def patched_models(monkeypatch):
deployment.litellm_params.model = "gpt-4"
router.get_deployment_by_model_group_name = MagicMock(return_value=deployment)
router.get_routable_upstream_model = MagicMock(return_value="gpt-4")
router.get_routable_upstream_provider = MagicMock(return_value=None)
router.get_configured_display_name = MagicMock(return_value=None)
router.get_configured_service_tiers = MagicMock(return_value=())
router.get_configured_max_input_tokens = MagicMock(return_value=())

2
tests/unit/proxy/router_entry_point_stubs.py
View file

@ -99,6 +99,8 @@ def route_entry_points_to_stubs(router: _RouterDouble) -> _RouterDouble:
return router.is_recognized_model(name)
if kind == "upstream":
return router.get_routable_upstream_model(name, **_given(team_id=team_id))
if kind == "upstream_provider":
return router.get_routable_upstream_provider(name, **_given(team_id=team_id))
if kind == "search_tool_support":
return router.get_search_tool_support(name, **_given(team_id=team_id))
if kind == "from_id":

21
tests/unit/test_router/test_router.py
View file

@ -13950,6 +13950,27 @@ def test_upstream_model_and_service_tiers_are_read_off_the_deployments_the_team_
assert router.get_configured_service_tiers(model_name, team_id) == service_tiers
@pytest.mark.parametrize(
("litellm_params", "provider"),
[
({"model": "us.openai.gpt-6.1-sol", "custom_llm_provider": "bedrock"}, "bedrock"),
({"model": "bedrock_mantle/openai.gpt-6.1-sol"}, "bedrock_mantle"),
({"model": "gpt-6.1-sol"}, None),
],
)
def test_the_upstream_provider_is_the_one_the_routable_deployment_declares(litellm_params, provider):
router = litellm.Router(
model_list=[
{"model_name": "sol", "litellm_params": litellm_params, "model_info": {"team_id": "team-1"}},
{"model_name": "sol", "litellm_params": {"model": "azure/sol-deployment"}, "model_info": {"team_id": "t2"}},
]
)
assert router.registry().resolve("sol", kind="upstream_provider", team_id="team-1") == provider
assert router.registry().resolve("sol", kind="upstream_provider", team_id="t2") == "azure"
assert router.registry().resolve("not-a-real-model", kind="upstream_provider") is None
def test_get_configured_display_name_treats_malformed_values_as_absent():
malformed = ["", " ", 12345, ["Kimi K3"], {"name": "Kimi K3"}, True]
router = litellm.Router(

5
tests/unit/test_router/test_router_entry_points.py
View file

@ -656,6 +656,11 @@ FORWARDS = [
"_core_get_routable_upstream_model",
(("gpt",), {"team_id": "t"}),
),
(
lambda r: r.registry().resolve("gpt", kind="upstream_provider", team_id="t"),
"_core_get_routable_upstream_provider",
(("gpt",), {"team_id": "t"}),
),
(
lambda r: r.registry().resolve("gpt", kind="search_tool_support", team_id="t"),
"_core_get_search_tool_support",