From d5922c4bae2a8e44f342ca46f923bc53d1de5c7f Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Tue, 3 Nov 2026 00:15:46 -0800 Subject: [PATCH] fix(proxy): keep Codex tiers for unpriced deployments and match stock entries by declared provider A deployment with no token price no longer votes a Codex service tier off, an undescribed model keeps Codex's default reasoning summary parameter, a Bedrock deployment naming its provider in custom_llm_provider gets Codex's stock entry, and Bedrock provider namespaces follow the AWS model id format. --- litellm/llms/bedrock/common_utils.py | 18 ++- .../proxy/common_utils/codex_model_catalog.py | 60 ++++++--- .../common_utils/codex_model_translation.py | 42 +++--- litellm/router_api.py | 24 +++- litellm/router_python_core.py | 41 ++++-- .../llms/bedrock/test_bedrock_common_utils.py | 11 +- tests/unit/proxy/client/cli/test_agents.py | 1 + .../common_utils/test_codex_model_catalog.py | 125 +++++++++++++----- .../test_codex_model_translation.py | 56 ++++---- .../proxy/proxy_server/test_routes_models.py | 1 + tests/unit/proxy/router_entry_point_stubs.py | 2 + tests/unit/test_router/test_router.py | 21 +++ .../test_router/test_router_entry_points.py | 5 + 13 files changed, 291 insertions(+), 116 deletions(-) diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 3c329504361..8a570938015 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -13,7 +13,7 @@ import warnings from collections.abc import Callable, Iterator, Mapping, Sequence from dataclasses import dataclass from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, get_args +from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias from typing_extensions import ( ReadOnly, @@ -32,7 +32,6 @@ from pydantic import ConfigDict, TypeAdapter, ValidationError import litellm from litellm import verbose_logger -from litellm.constants import BEDROCK_INVOKE_PROVIDERS_LITERAL from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix from litellm.litellm_core_utils.optional_imports import ensure_optional_import from litellm.litellm_core_utils.prompt_templates.common_utils import schema_property_count @@ -1491,12 +1490,17 @@ def get_bedrock_base_model(model: str) -> str: return model -def bedrock_vendor_model_name(model: str) -> str: - """The name the model's vendor publishes it under, without Bedrock's routing, region and vendor - prefixes: ``bedrock/us.openai.gpt-6.1-sol`` and ``us-east-1/openai.gpt-6.1-sol`` are ``gpt-6.1-sol``.""" +def bedrock_model_name_without_provider(model: str) -> str: + """The model id without Bedrock's routing and region prefixes and without its provider namespace. + + A Bedrock foundation model id is ``.`` with a one-word provider (``anthropic.``, + ``deepseek.``, ``moonshotai.``), so a base model whose first dot-separated part is one word drops it: + ``bedrock/us.openai.gpt-6.1-sol`` and ``us-east-1/openai.gpt-6.1-sol`` are ``gpt-6.1-sol``, + ``us.deepseek.r1-v1:0`` is ``r1-v1:0``. Any other base model, such as ``gpt-5.5`` or an imported model's + ARN id, is returned as is.""" base: Final = get_bedrock_base_model(model) - vendor, separator, name = base.partition(".") - return name if separator and vendor in get_args(BEDROCK_INVOKE_PROVIDERS_LITERAL) else base + provider, separator, name = base.partition(".") + return name if separator and provider.isalnum() else base def bedrock_converse_accepts_redacted_thinking(model: str, base_model: str | None) -> bool: diff --git a/litellm/proxy/common_utils/codex_model_catalog.py b/litellm/proxy/common_utils/codex_model_catalog.py index 715be694f96..2f8d17ea2d2 100644 --- a/litellm/proxy/common_utils/codex_model_catalog.py +++ b/litellm/proxy/common_utils/codex_model_catalog.py @@ -6,8 +6,8 @@ feature on) and decodes the body as its own ``ModelsResponse``, ``{"models": [Mo never the OpenAI list shape. Codex then ignores its own bundled catalog, so this body is all it knows about the provider's models. It is built from the rows ``/v1/models`` already lists: a model Codex knows, by its id or by the upstream model its deployments call once the provider's own prefixes are -stripped, keeps Codex's own entry for it, minus any service tier some deployment behind it has no -price for; any other model gets an entry translated from the capabilities its deployments' model info +stripped, keeps Codex's own entry for it, minus any service tier some deployment behind it prices tokens +but not that tier for; any other model gets an entry translated from the capabilities its deployments' model info declares (see ``codex_model_translation``). ``model_info.service_tiers`` becomes the entry's ``service_tiers``, which Codex turns into slash commands (``/ultrafast``) that send ``service_tier`` upstream. A ``model_info.max_input_tokens`` the operator sets replaces a known model's @@ -35,16 +35,18 @@ from typing import TYPE_CHECKING, Final, Literal, TypeAlias from pydantic import ConfigDict, TypeAdapter, ValidationError from litellm._logging import verbose_proxy_logger -from litellm.llms.bedrock.common_utils import bedrock_vendor_model_name +from litellm.litellm_core_utils.get_llm_provider_logic import declared_provider +from litellm.llms.bedrock.common_utils import bedrock_model_name_without_provider from litellm.proxy.common_utils.codex_model_translation import ( TIER_PRICE_KEYS, + TOKEN_PRICE_KEY, CodexDeploymentCapabilities, CodexFallbackModel, CodexServiceTier, TierId, codex_fallback_entry, combined_capabilities, - every_deployment_prices, + no_priced_deployment_lacks_tier, ) from litellm.types.llms.base_model import LiteLLMBaseModel @@ -122,7 +124,8 @@ CodexAliasDefaults: TypeAlias = Literal["codex", "deployment"] @dataclass(frozen=True, slots=True) class CodexCatalogRow: """What the catalog needs to know about one listed model: its public id, the listing's mode and - input limit, the upstream model behind it, the reasoning effort its deployments pin, and the + input limit, the upstream model behind it and the provider its deployment declares, the reasoning + effort its deployments pin, and the `model_info` values the operator set; `service_tiers` holds one raw `model_info.service_tiers` value per deployment behind the id, None where a deployment sets none, and `configured_max_input_tokens` one `model_info.max_input_tokens` per deployment the same way. @@ -133,6 +136,7 @@ class CodexCatalogRow: mode: str | None = None max_input_tokens: int | None = None upstream_model: str | None = None + upstream_provider: str | None = None reasoning_effort: str | None = None display_name: str | None = None service_tiers: tuple[object, ...] = () @@ -242,36 +246,44 @@ def _codex_can_drive(model_id: str, mode: str | None) -> bool: return "*" not in model_id and (mode is None or mode in CODEX_CHAT_MODES) -_VENDOR_MODEL_NAMES: Final[Mapping[str, Callable[[str], str]]] = MappingProxyType( - {"bedrock": bedrock_vendor_model_name, "bedrock_mantle": bedrock_vendor_model_name} +_PROVIDER_MODEL_NAMES: Final[Mapping[str, Callable[[str], str]]] = MappingProxyType( + {"bedrock": bedrock_model_name_without_provider, "bedrock_mantle": bedrock_model_name_without_provider} ) -def _vendor_model_name(upstream_model: str) -> str: - provider, _, model = upstream_model.partition("/") - name_for: Final = _VENDOR_MODEL_NAMES.get(provider) +def _provider_model_name(upstream_model: str, upstream_provider: str | None) -> str: + """The upstream model without the prefixes its provider adds, the provider being the one the deployment + declares in `custom_llm_provider`, else the model's `provider/` prefix.""" + provider: Final = declared_provider(upstream_model, upstream_provider) + if provider is None: + return upstream_model + model: Final = upstream_model.removeprefix(f"{provider}/") + name_for: Final = _PROVIDER_MODEL_NAMES.get(provider) return model if name_for is None else name_for(model) def _stock_for(row: CodexCatalogRow, stock: Mapping[str, CodexStockModel]) -> CodexStockModel | None: """Codex's entry for the row's id, else for its upstream model as written, else for that model's name - once the provider's own prefixes are stripped (`bedrock/us.openai.gpt-6.1-sol` is `gpt-6.1-sol`).""" + once its provider's own prefixes are stripped (`bedrock/us.openai.gpt-6.1-sol` is `gpt-6.1-sol`).""" names: Final = ( (row.id,) if row.upstream_model is None - else (row.id, row.upstream_model, _vendor_model_name(row.upstream_model)) + else (row.id, row.upstream_model, _provider_model_name(row.upstream_model, row.upstream_provider)) ) return next((stock[name] for name in names if name in stock), None) def _priced_tiers(row: CodexCatalogRow, stock_model: CodexStockModel) -> Mapping[str, object]: - """Codex's own tiers for the model without the ones some deployment behind the row has no price for, - or nothing when every deployment prices them all. Sending a tier the upstream does not sell fails the - request, and the cost map prices a tier for exactly the models that sell it.""" + """Codex's own tiers for the model without the ones some deployment behind the row prices tokens but not + that tier for, or nothing when none is dropped. Sending a tier the upstream does not sell fails the + request, and the cost map prices a tier for exactly the models that sell it, while a deployment it has no + token price for at all says nothing either way.""" deployments: Final = row.deployment_capabilities - tiers: Final = tuple(tier for tier in stock_model.service_tiers if every_deployment_prices(tier.id, deployments)) + tiers: Final = tuple( + tier for tier in stock_model.service_tiers if no_priced_deployment_lacks_tier(tier.id, deployments) + ) speed_tiers: Final = tuple( - tier for tier in stock_model.additional_speed_tiers if every_deployment_prices(tier, deployments) + tier for tier in stock_model.additional_speed_tiers if no_priced_deployment_lacks_tier(tier, deployments) ) if tiers == stock_model.service_tiers and speed_tiers == stock_model.additional_speed_tiers: return _NO_UPDATE @@ -457,8 +469,8 @@ def codex_models_response_json( `max_context_window`. A row listed under another name than the Codex model its `upstream_model` names (an alias) takes that model's entry; `alias_defaults="deployment"` drops the defaults Codex picks for that model in favor of what the alias's deployments pin, while `"codex"` keeps them, as Codex would use them for the - model's own name. Codex's own service tiers stay on a known model's entry only when every deployment - behind the row prices them, and only while the operator configures none. `client_version` is the + model's own name. Codex's own service tiers stay on a known model's entry only when no deployment + behind the row prices tokens without pricing them, and only while the operator configures none. `client_version` is the requesting Codex's version, which decides the reasoning levels a fallback entry may offer; None counts as a version too old for custom levels. """ @@ -492,14 +504,17 @@ def deployment_capabilities(model_info: Mapping[str, object] | None, model_id: s Model info that does not validate declares nothing, logged, so one bad value never fails the listing.""" if model_info is None: return CodexDeploymentCapabilities() + prices_tokens: Final = model_info.get(TOKEN_PRICE_KEY) is not None priced: Final = frozenset(key for key in TIER_PRICE_KEYS if model_info.get(key) is not None) try: - return CodexDeploymentCapabilities.model_validate({**model_info, "priced_service_tier_keys": priced}) + return CodexDeploymentCapabilities.model_validate( + {**model_info, "prices_tokens": prices_tokens, "priced_service_tier_keys": priced} + ) except ValidationError as e: verbose_proxy_logger.warning( "model info of a deployment of %s declares no capability to Codex: %s", model_id, e.errors()[0]["msg"] ) - return CodexDeploymentCapabilities(priced_service_tier_keys=priced) + return CodexDeploymentCapabilities(prices_tokens=prices_tokens, priced_service_tier_keys=priced) def _deployment_capabilities( @@ -522,6 +537,9 @@ def _catalog_row( mode=row.get("mode"), max_input_tokens=row.get("max_input_tokens"), upstream_model=registry.resolve(lookup_id, kind="upstream", team_id=team_id) if registry is not None else None, + upstream_provider=( + registry.resolve(lookup_id, kind="upstream_provider", team_id=team_id) if registry is not None else None + ), display_name=registry.group(label_id, field="display_name") if registry is not None else None, service_tiers=registry.group(lookup_id, field="service_tiers", team_id=team_id) if registry is not None else (), supported_reasoning_efforts=_group_reasoning_efforts(row, lookup_id, registry, team_id), diff --git a/litellm/proxy/common_utils/codex_model_translation.py b/litellm/proxy/common_utils/codex_model_translation.py index 1ce4a7a65d1..770088d16ac 100644 --- a/litellm/proxy/common_utils/codex_model_translation.py +++ b/litellm/proxy/common_utils/codex_model_translation.py @@ -26,14 +26,18 @@ CodexInputModality: TypeAlias = Literal["text", "image"] _TEXT_ONLY: Final[tuple[CodexInputModality, ...]] = ("text",) _TEXT_AND_IMAGE: Final[tuple[CodexInputModality, ...]] = ("text", "image") -_TIER_PRICE_BASE_KEY: Final = "input_cost_per_token" -TIER_PRICE_KEYS: Final = tuple(f"{_TIER_PRICE_BASE_KEY}{suffix}" for suffix in SERVICE_TIER_COST_KEY_SUFFIXES) +TOKEN_PRICE_KEY: Final = "input_cost_per_token" +TIER_PRICE_KEYS: Final = tuple(f"{TOKEN_PRICE_KEY}{suffix}" for suffix in SERVICE_TIER_COST_KEY_SUFFIXES) _FIRST_CODEX_WITH_CUSTOM_EFFORTS: Final = Version("0.138.0") _EFFORTS_EVERY_CODEX_PARSES: Final = frozenset({"none", "minimal", "low", "medium", "high", "xhigh"}) _PREFERRED_DEFAULT_EFFORT: Final = "medium" _CODEX_UNSET_EFFORT: Final = "none" +def _unset(value: object) -> bool: + return value is None + + class CodexServiceTier(LiteLLMBaseModel): """A `ModelServiceTier` as Codex reads it: the slash command is `name` lowercased, and toggling it sends `id` as the request's `service_tier`.""" @@ -75,7 +79,7 @@ class CodexFallbackModel(LiteLLMBaseModel): slug: str display_name: str description: NoneType = None - default_reasoning_level: str | None = Field(default=None, exclude_if=lambda level: level is None) + default_reasoning_level: str | None = Field(default=None, exclude_if=_unset) supported_reasoning_levels: tuple[CodexFallbackReasoningLevel, ...] = () shell_type: Literal["unified_exec"] = "unified_exec" visibility: Literal["list"] = "list" @@ -85,7 +89,7 @@ class CodexFallbackModel(LiteLLMBaseModel): availability_nux: NoneType = None upgrade: NoneType = None support_verbosity: Literal[False] = False - supports_reasoning_summary_parameter: bool = False + supports_reasoning_summary_parameter: bool | None = Field(default=None, exclude_if=_unset) supports_reasoning_summaries: bool = False supports_parallel_tool_calls: bool = False default_verbosity: NoneType = None @@ -101,7 +105,8 @@ class CodexFallbackModel(LiteLLMBaseModel): class CodexDeploymentCapabilities(LiteLLMBaseModel): """What one deployment's model info says about the capabilities a Codex entry describes, None where - it says nothing, and the input price keys of the service tiers it has a price for.""" + it says nothing, whether it has a token price at all, and the input price keys of the service tiers it + has a price for.""" model_config = ConfigDict(frozen=True, extra="ignore") @@ -110,14 +115,19 @@ class CodexDeploymentCapabilities(LiteLLMBaseModel): supports_function_calling: bool | None = None supports_parallel_function_calling: bool | None = None supports_reasoning: bool | None = None + prices_tokens: bool = False priced_service_tier_keys: frozenset[str] = frozenset() -def every_deployment_prices(tier_id: str, deployments: Sequence[CodexDeploymentCapabilities]) -> bool: - """Whether every deployment has a price for `tier_id`, the way cost tracking prices a request sent at - that tier; a tier cost tracking has no price key for is priced by none.""" - price_key: Final = get_service_tier_cost_key(_TIER_PRICE_BASE_KEY, tier_id) - return all(price_key in deployment.priced_service_tier_keys for deployment in deployments) +def no_priced_deployment_lacks_tier(tier_id: str, deployments: Sequence[CodexDeploymentCapabilities]) -> bool: + """Whether every deployment that has a token price also has one for `tier_id`, the way cost tracking + prices a request sent at that tier. A deployment with no token price at all (a subscription upstream, + a deployment name the cost map does not know) says nothing about tiers, so it does not vote, and a tier + cost tracking has no price key for is lacked by every deployment that prices tokens.""" + price_key: Final = get_service_tier_cost_key(TOKEN_PRICE_KEY, tier_id) + return all( + price_key in deployment.priced_service_tier_keys for deployment in deployments if deployment.prices_tokens + ) @dataclass(frozen=True, slots=True) @@ -205,12 +215,12 @@ def codex_fallback_entry( ) -> CodexFallbackModel: """Codex's entry for a model it does not know, from the capabilities its deployments share. - A capability no deployment declares keeps Codex's own default for an unknown model, except the two - that send something a model may reject: the `reasoning.summary` parameter goes out only to a model - that reasons, and the freeform `apply_patch` tool only to one that calls tools. + A capability no deployment declares keeps Codex's own default for an unknown model, except the freeform + `apply_patch` tool, which goes out only to a model that calls tools: `supports_reasoning_summary_parameter` + is left out, which Codex reads as true, and the fields only Codex before 0.150.0 reads keep the false + it requires. """ levels, default_level = codex_reasoning_levels(reasoning_efforts, client_version) - reasons: Final = capabilities.supports_reasoning is True return CodexFallbackModel( slug=slug, display_name=display_name, @@ -218,8 +228,8 @@ def codex_fallback_entry( supported_reasoning_levels=levels, priority=priority, service_tiers=service_tiers, - supports_reasoning_summary_parameter=reasons, - supports_reasoning_summaries=reasons, + supports_reasoning_summary_parameter=capabilities.supports_reasoning, + supports_reasoning_summaries=capabilities.supports_reasoning is True, supports_parallel_tool_calls=capabilities.supports_parallel_function_calling is True, apply_patch_tool_type="freeform" if capabilities.supports_function_calling is True else None, input_modalities=_TEXT_ONLY if capabilities.supports_vision is False else _TEXT_AND_IMAGE, diff --git a/litellm/router_api.py b/litellm/router_api.py index 915a5f34d39..acf4d61e207 100644 --- a/litellm/router_api.py +++ b/litellm/router_api.py @@ -143,6 +143,7 @@ ResolveKind: TypeAlias = Literal[ "team_model", "recognized", "upstream", + "upstream_provider", "group_of_id", "wildcard_access_group", "team_model_alternatives", @@ -685,7 +686,7 @@ class RouterRegistry: self, name: str, *, - kind: Literal["alias", "upstream"], + kind: Literal["alias", "upstream", "upstream_provider"], team_id: str | None = None, custom_llm_provider: str | None = None, served_model: str | None = None, @@ -765,6 +766,8 @@ class RouterRegistry: - ``team_model``: a team's public name to its group (was ``map_team_model``; ``team_id`` required) - ``recognized``: whether the router knows the name (was ``is_recognized_model``) - ``upstream``: the upstream model a name routes to (was ``get_routable_upstream_model``) + - ``upstream_provider``: the provider the deployment ``upstream`` reads declares, by its + ``custom_llm_provider`` or its model's ``provider/`` prefix - ``group_of_id``: the group of a deployment id (was ``get_model_group``) - ``wildcard_access_group``: whether an access group grants a wildcard route (was ``is_model_access_group_for_wildcard_route``) @@ -794,10 +797,8 @@ class RouterRegistry: return name if served_model is None else r._core_resolve_served_model_name(name, served_model) if kind == "recognized": return r._core_is_recognized_model(name) - if kind == "upstream": - return r._core_get_routable_upstream_model(name, team_id=team_id) - if kind == "search_tool_support": - return r._core_get_search_tool_support(name, team_id=team_id) + if kind in ("upstream", "upstream_provider", "search_tool_support"): + return self._resolve_routable(name, kind, team_id) if kind == "group_of_id": return r._core_get_model_group(id=name) if kind == "wildcard_access_group": @@ -806,6 +807,19 @@ class RouterRegistry: return r._core_team_model_has_alternatives(name) raise ValueError(f"unknown resolve kind {kind!r}") + def _resolve_routable( + self, + name: str, + kind: Literal["upstream", "upstream_provider", "search_tool_support"], + team_id: str | None, + ) -> str | tuple[bool | None, ...] | None: + r: Final = self._router + if kind == "upstream": + return r._core_get_routable_upstream_model(name, team_id=team_id) + if kind == "upstream_provider": + return r._core_get_routable_upstream_provider(name, team_id=team_id) + return r._core_get_search_tool_support(name, team_id=team_id) + def aliases( self, ) -> "dict[str, str | RouterModelGroupAliasItem]": # mutable-ok: the model_group_alias attribute, the live dict diff --git a/litellm/router_python_core.py b/litellm/router_python_core.py index c24d5a16ce0..ed9c9e72c13 100644 --- a/litellm/router_python_core.py +++ b/litellm/router_python_core.py @@ -137,6 +137,7 @@ from litellm.litellm_core_utils.deployment_pricing import deployment_pricing_mod from litellm.litellm_core_utils.get_litellm_params import SUPPLEMENTAL_KWARGS_KEYS from litellm.litellm_core_utils.get_llm_provider_logic import ( declared_authenticating_provider, + declared_provider, inferred_provider, is_registered_custom_provider, resolve_model_and_provider_for_metadata, @@ -786,6 +787,16 @@ def _operator_max_input_tokens(deployment: Mapping[str, object]) -> int | None: return coerce_token_limit(model_info.get("max_input_tokens")) +def _deployment_params(deployment: DeploymentTypedDict) -> Mapping[str, object]: + return deployment["litellm_params"] + + +def _model_info_key(deployment: DeploymentTypedDict) -> tuple[str, str] | None: + deployment_id: Final = _configured_model_info(deployment).get("id") + model: Final = _deployment_params(deployment).get("model") + return (deployment_id, model) if isinstance(deployment_id, str) and isinstance(model, str) else None + + def _search_tool_support(deployment: DeploymentTypedDict) -> bool | None: from litellm.responses.litellm_completion_transformation.tool_search import chat_config_builds_anthropic_request @@ -12753,15 +12764,10 @@ class PythonCore(DeprecatedRouterMembers): Returns an empty tuple for wildcard-expanded or unknown names. """ - return tuple( - self._core_cached_deployment_model_info(deployment_id, model) - if isinstance(deployment_id := _configured_model_info(deployment).get("id"), str) - and isinstance( - model := _LITELLM_PARAMS_ADAPTER.validate_python(deployment["litellm_params"]).get("model"), str - ) - else None - for deployment in self._routable_deployments(model_name, team_id) + keys: Final = tuple( + _model_info_key(deployment) for deployment in self._routable_deployments(model_name, team_id) ) + return tuple(None if key is None else self._core_cached_deployment_model_info(*key) for key in keys) def _core_get_routable_upstream_model(self, model_name: str, team_id: str | None = None) -> str | None: """ @@ -12774,6 +12780,25 @@ class PythonCore(DeprecatedRouterMembers): deployment: Final = next(iter(self._routable_deployments(model_name, team_id)), None) return deployment["litellm_params"].get("model") if deployment is not None else None + def _core_get_routable_upstream_provider(self, model_name: str, team_id: str | None = None) -> str | None: + """ + Return the provider the deployment _core_get_routable_upstream_model + reads declares: its `litellm_params.custom_llm_provider`, else the + `provider/` prefix of its `litellm_params.model`. + + Returns None when that deployment declares neither, and wherever + _core_get_routable_upstream_model returns None. + """ + deployment: Final = next(iter(self._routable_deployments(model_name, team_id)), None) + if deployment is None: + return None + params: Final = _deployment_params(deployment) + model: Final = params.get("model") + provider: Final = params.get("custom_llm_provider") + return declared_provider( + model if isinstance(model, str) else None, provider if isinstance(provider, str) else None + ) + def _core_get_credential_deployment(self, model_id: str, team_id: str | None = None) -> Deployment | None: """ The deployment a passthrough endpoint (files, batches, etc.) resolves for a diff --git a/tests/unit/llms/bedrock/test_bedrock_common_utils.py b/tests/unit/llms/bedrock/test_bedrock_common_utils.py index 8c047847b6c..bd3f17eea51 100644 --- a/tests/unit/llms/bedrock/test_bedrock_common_utils.py +++ b/tests/unit/llms/bedrock/test_bedrock_common_utils.py @@ -9,7 +9,7 @@ from litellm.llms.bedrock.common_utils import( NativeOutputFallback, bedrock_native_output_is_opt_in, bedrock_strict_tools_fallback, - bedrock_vendor_model_name, + bedrock_model_name_without_provider, deployment_model_info, extract_model_name_from_bedrock_arn, get_bedrock_base_model, @@ -312,12 +312,15 @@ def test_legacy_mantle_route_prefix_stripped_for_cost_lookup(): ("us-east-1/openai.gpt-6.1-sol", "gpt-6.1-sol"), ("openai.gpt-5.5", "gpt-5.5"), ("arn:aws:bedrock:us-east-1:123456789012:inference-profile/us.meta.llama4-scout-17b", "llama4-scout-17b"), + ("us.deepseek.r1-v1:0", "r1-v1:0"), + ("moonshotai.kimi-k2-thinking", "kimi-k2-thinking"), + ("bedrock/qwen.qwen3-32b-v1:0", "qwen3-32b-v1:0"), ("gpt-5.5", "gpt-5.5"), - ("unknownvendor.some-model", "unknownvendor.some-model"), + ("arn:aws:bedrock:us-east-1:123456789012:imported-model/abc123", "abc123"), ], ) -def test_bedrock_vendor_model_name_strips_routing_region_and_vendor_prefixes_only(model, expected): - assert bedrock_vendor_model_name(model) == expected +def test_bedrock_model_name_without_provider_strips_routing_region_and_provider_prefixes_only(model, expected): + assert bedrock_model_name_without_provider(model) == expected def test_output_config_effort_normalization_uses_model_info_ceiling(monkeypatch): diff --git a/tests/unit/proxy/client/cli/test_agents.py b/tests/unit/proxy/client/cli/test_agents.py index 12954ab7304..a6de6a7ee50 100644 --- a/tests/unit/proxy/client/cli/test_agents.py +++ b/tests/unit/proxy/client/cli/test_agents.py @@ -1041,6 +1041,7 @@ class TestCodexModelSync: assert entry["experimental_supported_tools"] == [] assert entry["support_verbosity"] is False assert entry["supports_reasoning_summaries"] is False + assert "supports_reasoning_summary_parameter" not in entry assert entry["supports_parallel_tool_calls"] is False for nullable in ("description", "availability_nux", "upgrade", "default_verbosity", "apply_patch_tool_type"): assert nullable in entry and entry[nullable] is None diff --git a/tests/unit/proxy/common_utils/test_codex_model_catalog.py b/tests/unit/proxy/common_utils/test_codex_model_catalog.py index 61c343c7444..4ab4c24e5b3 100644 --- a/tests/unit/proxy/common_utils/test_codex_model_catalog.py +++ b/tests/unit/proxy/common_utils/test_codex_model_catalog.py @@ -87,29 +87,10 @@ def test_body_is_exactly_codex_models_response(): assert [entry["slug"] for entry in body["models"]] == ["gpt-6-astra"] -def test_unknown_model_entry_carries_every_field_codex_deserializes_without_a_default(): - """Codex 0.166.0 `codex-rs/protocol/src/openai_models.rs` ModelInfo, the fields with no - `#[serde(default)]`, read on 2026-11-02; `base_instructions` because its `ModelsResponse` - decoder rejects an entry with neither it nor `model_messages.instructions_template`.""" +def test_unknown_model_entry_takes_its_slug_window_and_prompt_from_the_row(): + """The fields every Codex release decodes are pinned in test_codex_model_translation.py.""" (entry,) = _models(_row("gpt-6-astra")) - assert entry.keys() >= { - "slug", - "display_name", - "description", - "supported_reasoning_levels", - "shell_type", - "visibility", - "supported_in_api", - "priority", - "availability_nux", - "upgrade", - "support_verbosity", - "default_verbosity", - "apply_patch_tool_type", - "truncation_policy", - "experimental_supported_tools", - } assert (entry["slug"], entry["display_name"], entry["visibility"], entry["supported_in_api"]) == ( "gpt-6-astra", "gpt-6-astra", @@ -722,6 +703,7 @@ def test_catalog_rows_read_the_router_by_each_entry_lookup_id(): mode="chat", max_input_tokens=7, upstream_model="openai/gpt-5.5", + upstream_provider="openai", display_name="Team 5.5", service_tiers=(["ultrafast"], None), supported_reasoning_efforts=router.get_model_group_info( @@ -733,6 +715,7 @@ def test_catalog_rows_read_the_router_by_each_entry_lookup_id(): CodexCatalogRow( id="plain", upstream_model="openai/some-unmapped-model", + upstream_provider="openai", service_tiers=(None,), search_tool_support=(None,), configured_max_input_tokens=(None,), @@ -766,6 +749,7 @@ def test_catalog_rows_read_an_alias_off_its_target_under_the_alias_name(): mode="chat", max_input_tokens=9, upstream_model="openai/gpt-6-astra", + upstream_provider="openai", service_tiers=(["ultrafast"],), supported_reasoning_efforts=router.get_model_group_info("gpt-6-astra").supported_reasoning_efforts, search_tool_support=(None,), @@ -974,6 +958,41 @@ def test_a_model_codex_knows_keeps_codexs_entry_under_any_provider_prefix(upstre assert entry["supported_reasoning_levels"] == stock["supported_reasoning_levels"] +@pytest.mark.parametrize("model", ["us.openai.{slug}", "converse/global.openai.{slug}"]) +def test_a_bedrock_deployment_naming_its_provider_apart_from_the_model_keeps_codexs_entry(model: str) -> None: + router: Final = litellm.Router( + model_list=[ + { + "model_name": "team-model", + "litellm_params": { + "model": model.format(slug=_SOL_SLUG), + "custom_llm_provider": "bedrock", + "aws_region_name": "us-east-1", + }, + } + ] + ) + + entry: Final = _served(router, "team-model")["team-model"] + + assert entry["model_messages"] == bundled_codex_models()[_SOL_SLUG].model_dump(mode="json")["model_messages"] + + +def test_an_upstream_with_no_provider_prefix_matches_only_by_its_own_name() -> None: + (unknown, known) = json.loads( + codex_models_response_json( + ( + _row("first", upstream_model=f"us.openai.{_SOL_SLUG}"), + _row("second", upstream_model=_SOL_SLUG), + ), + instructions=_PROMPT, + ).json + )["models"] + + assert "model_messages" not in unknown and unknown["base_instructions"] == _PROMPT + assert known["model_messages"] == bundled_codex_models()[_SOL_SLUG].model_dump(mode="json")["model_messages"] + + @pytest.mark.parametrize( "upstream", ["bedrock/us.openai.{slug}-mini", "bedrock/us.meta.{slug}x", "bedrock_mantle/openai.{slug}.1", "{slug}-0"], @@ -1002,7 +1021,9 @@ def _listing(*names: str, max_input_tokens: int | None = None) -> tuple[dict[str ) -def _served(router: litellm.Router, *names: str, team_id: str | None = None, **listing: int) -> dict[str, dict]: +def _served( + router: litellm.Router, *names: str, team_id: str | None = None, **listing: int +) -> dict[str, Mapping[str, object]]: body: Final = codex_model_list_body( _listing(*names, **listing), tuple((name, name) for name in names), router, team_id, client_version="0.166.0" ) @@ -1077,20 +1098,20 @@ def test_a_claude_like_deployment_gets_codexs_coding_setup() -> None: supports_reasoning=True, ), _vllm("plain", "h12"), + _vllm("no-reasoning", "h12b", supports_reasoning=False), ] ) - served: Final = _served(router, "claude-like", "plain") + served: Final = _served(router, "claude-like", "plain", "no-reasoning") claude_like: Final = served["claude-like"] assert claude_like["apply_patch_tool_type"] == "freeform" assert claude_like["truncation_policy"] == {"mode": "tokens", "limit": 10000} assert claude_like["supports_reasoning_summary_parameter"] is True assert claude_like["context_window"] == 1_000_000 - assert (served["plain"]["apply_patch_tool_type"], served["plain"]["supports_reasoning_summary_parameter"]) == ( - None, - False, - ) + assert served["plain"]["apply_patch_tool_type"] is None + assert "supports_reasoning_summary_parameter" not in served["plain"] + assert served["no-reasoning"]["supports_reasoning_summary_parameter"] is False def test_capabilities_are_read_only_off_the_deployments_the_key_team_routes_to() -> None: @@ -1112,7 +1133,8 @@ def _tier_fields(entry: Mapping[str, object]) -> tuple[object, object, object]: def test_codexs_own_tiers_stay_only_when_every_deployment_behind_the_name_prices_them() -> None: stock: Final = bundled_codex_models()[_SOL_SLUG].model_dump(mode="json") assert stock["service_tiers"] and stock["additional_speed_tiers"] and stock["default_service_tier"] - priority_price: Final = {"input_cost_per_token_priority": 1e-05} + token_price: Final = {"input_cost_per_token": 1e-06} + priority_price: Final = {**token_price, "input_cost_per_token_priority": 1e-05} def upstream(host: str) -> dict[str, str]: return {"model": f"hosted_vllm/{_SOL_SLUG}", "api_base": f"https://{host}.example/v1"} @@ -1122,7 +1144,9 @@ def test_codexs_own_tiers_stay_only_when_every_deployment_behind_the_name_prices {"model_name": "all-priced", "litellm_params": upstream("a"), "model_info": priority_price}, {"model_name": "all-priced", "litellm_params": upstream("b"), "model_info": priority_price}, {"model_name": "half-priced", "litellm_params": upstream("c"), "model_info": priority_price}, - {"model_name": "half-priced", "litellm_params": upstream("d")}, + {"model_name": "half-priced", "litellm_params": upstream("d"), "model_info": token_price}, + {"model_name": "priced-and-unknown", "litellm_params": upstream("f"), "model_info": priority_price}, + {"model_name": "priced-and-unknown", "litellm_params": {"model": "azure/prod-sol-eastus"}}, { "model_name": "configured", "litellm_params": upstream("e"), @@ -1131,14 +1155,55 @@ def test_codexs_own_tiers_stay_only_when_every_deployment_behind_the_name_prices ] ) - served: Final = _served(router, "all-priced", "half-priced", "configured") + served: Final = _served(router, "all-priced", "half-priced", "priced-and-unknown", "configured") assert _tier_fields(served["all-priced"]) == _tier_fields(stock) + assert _tier_fields(served["priced-and-unknown"]) == _tier_fields(stock) assert _tier_fields(served["half-priced"]) == ([], [], None) assert served["half-priced"]["model_messages"] == stock["model_messages"] assert [tier["id"] for tier in served["configured"]["service_tiers"]] == ["priority"] +def _tiered_stock(default_service_tier: str) -> Mapping[str, CodexStockModel]: + tiers: Final = ( + CodexServiceTier(id="priority", name="Fast", description="faster"), + CodexServiceTier(id="flex", name="Flex", description="cheaper"), + ) + return MappingProxyType( + { + "tiered": CodexStockModel( + slug="tiered", + display_name="Tiered", + priority=0, + visibility="list", + service_tiers=tiers, + default_service_tier=default_service_tier, + ) + } + ) + + +@pytest.mark.parametrize(("default_tier", "kept_default"), [("priority", "priority"), ("flex", None)]) +def test_dropping_one_stock_tier_keeps_the_default_only_while_it_is_still_offered(default_tier, kept_default) -> None: + priority_and_flex: Final = CodexDeploymentCapabilities( + prices_tokens=True, + priced_service_tier_keys=frozenset( + get_service_tier_cost_key("input_cost_per_token", tier) for tier in ("priority", "flex") + ), + ) + priority_only: Final = CodexDeploymentCapabilities( + prices_tokens=True, + priced_service_tier_keys=frozenset({get_service_tier_cost_key("input_cost_per_token", "priority")}), + ) + row: Final = _row("tiered", deployment_capabilities=(priority_and_flex, priority_only)) + + (entry,) = json.loads( + codex_models_response_json((row,), stock=_tiered_stock(default_tier), instructions=_PROMPT).json + )["models"] + + assert [tier["id"] for tier in entry["service_tiers"]] == ["priority"] + assert entry["default_service_tier"] == kept_default + def test_model_info_that_does_not_validate_declares_no_capability_but_keeps_its_prices() -> None: priced: Final = get_service_tier_cost_key("input_cost_per_token", "priority") diff --git a/tests/unit/proxy/common_utils/test_codex_model_translation.py b/tests/unit/proxy/common_utils/test_codex_model_translation.py index 4c23c65dea3..10fdfbd282e 100644 --- a/tests/unit/proxy/common_utils/test_codex_model_translation.py +++ b/tests/unit/proxy/common_utils/test_codex_model_translation.py @@ -1,5 +1,4 @@ import json -from itertools import product from typing import Final import pytest @@ -11,11 +10,12 @@ from litellm.proxy.common_utils.codex_model_translation import ( CodexServiceTier, codex_fallback_entry, combined_capabilities, - every_deployment_prices, + no_priced_deployment_lacks_tier, ) _PROMPT: Final = "base prompt" _SILENT: Final = CodexDeploymentCapabilities() +_ABSENT: Final = object() def _entry(capabilities: CodexCapabilities, **kwargs: object) -> dict[str, object]: @@ -42,21 +42,21 @@ def test_a_model_that_calls_tools_and_reasons_gets_codexs_own_coding_setup(): @pytest.mark.parametrize( - ("flag", "field", "when_true", "otherwise"), + ("flag", "field", "by_value"), [ - ("supports_function_calling", "apply_patch_tool_type", "freeform", None), - ("supports_reasoning", "supports_reasoning_summary_parameter", True, False), - ("supports_reasoning", "supports_reasoning_summaries", True, False), - ("supports_parallel_function_calling", "supports_parallel_tool_calls", True, False), + ("supports_function_calling", "apply_patch_tool_type", {True: "freeform", False: None, None: None}), + ("supports_reasoning", "supports_reasoning_summary_parameter", {True: True, False: False, None: _ABSENT}), + ("supports_reasoning", "supports_reasoning_summaries", {True: True, False: False, None: False}), + ("supports_parallel_function_calling", "supports_parallel_tool_calls", {True: True, False: False, None: False}), ], ) @pytest.mark.parametrize("value", [True, False, None]) -def test_a_feature_that_sends_something_upstream_is_on_only_when_every_deployment_declares_it( - flag, field, when_true, otherwise, value -): +def test_each_codex_feature_follows_what_the_deployments_declare(flag, field, by_value, value): + """A missing `supports_reasoning_summary_parameter` is Codex's own default (true, `default_true` in + `codex-rs/protocol/src/openai_models.rs` at rust-v0.166.0), so a model nobody describes keeps it.""" entry: Final = _entry(CodexCapabilities(**{flag: value})) - assert entry[field] == (when_true if value is True else otherwise) + assert entry.get(field, _ABSENT) == by_value[value] @pytest.mark.parametrize( @@ -124,23 +124,22 @@ _CODEX_MODALITIES: Final = {"text", "image"} _CODEX_TRUNCATION_MODES: Final = {"bytes", "tokens"} -@pytest.mark.parametrize( - "flags", list(product((True, False, None), repeat=4)), ids=lambda flags: "-".join(map(str, flags)) -) -def test_every_entry_carries_the_fields_and_enum_values_every_supported_codex_decodes(flags): +@pytest.mark.parametrize("described", [True, False], ids=["described", "undescribed"]) +def test_every_entry_carries_the_fields_and_enum_values_every_supported_codex_decodes(described): """`codex-rs/protocol/src/openai_models.rs` ModelInfo at rust-v0.140.0 and rust-v0.166.0, the fields with no `#[serde(default)]` and the variants of InputModality, ConfigShellToolType, ModelVisibility, ApplyPatchToolType and TruncationMode in both, read on 2026-11-02. Codex rejects the whole catalog when one entry does not decode.""" - vision, tools, parallel, reasoning = flags entry: Final = _entry( CodexCapabilities( context_window=128_000, - supports_vision=vision, - supports_function_calling=tools, - supports_parallel_function_calling=parallel, - supports_reasoning=reasoning, + supports_vision=False, + supports_function_calling=True, + supports_parallel_function_calling=True, + supports_reasoning=True, ) + if described + else CodexCapabilities() ) assert entry.keys() >= _REQUIRED_SINCE_CODEX_0_140 @@ -198,21 +197,28 @@ def test_the_window_is_the_smallest_any_deployment_declares(windows, listed, exp def _priced(*tier_ids: str) -> CodexDeploymentCapabilities: return CodexDeploymentCapabilities( - priced_service_tier_keys=frozenset(get_service_tier_cost_key("input_cost_per_token", tier) for tier in tier_ids) + prices_tokens=True, + priced_service_tier_keys=frozenset( + get_service_tier_cost_key("input_cost_per_token", tier) for tier in tier_ids + ), ) @pytest.mark.parametrize( - ("deployments", "tier", "priced"), + ("deployments", "tier", "kept"), [ ((_priced("priority"), _priced("priority", "flex")), "priority", True), ((_priced("priority"), _priced("flex")), "priority", False), - ((_priced("priority"), _SILENT), "priority", False), + ((_priced("priority"), _priced()), "priority", False), + ((_priced("priority"), _SILENT), "priority", True), + ((_SILENT, _SILENT), "priority", True), ((_priced("priority"),), "fast", True), ((_priced("flex"),), "fast", False), ((_priced("priority", "flex"),), "made-up-tier", False), ((), "priority", True), ], ) -def test_a_tier_is_priced_only_when_every_deployment_prices_it_the_way_cost_tracking_does(deployments, tier, priced): - assert every_deployment_prices(tier, deployments) is priced +def test_a_tier_stays_unless_a_deployment_that_prices_tokens_has_no_price_for_it(deployments, tier, kept): + """A deployment with no token price at all (a subscription upstream, a deployment name the cost map does + not know) says nothing about which tiers its upstream sells, so it does not vote.""" + assert no_priced_deployment_lacks_tier(tier, deployments) is kept diff --git a/tests/unit/proxy/proxy_server/test_routes_models.py b/tests/unit/proxy/proxy_server/test_routes_models.py index 189d669890b..06b50735cfb 100644 --- a/tests/unit/proxy/proxy_server/test_routes_models.py +++ b/tests/unit/proxy/proxy_server/test_routes_models.py @@ -56,6 +56,7 @@ def patched_models(monkeypatch): deployment.litellm_params.model = "gpt-4" router.get_deployment_by_model_group_name = MagicMock(return_value=deployment) router.get_routable_upstream_model = MagicMock(return_value="gpt-4") + router.get_routable_upstream_provider = MagicMock(return_value=None) router.get_configured_display_name = MagicMock(return_value=None) router.get_configured_service_tiers = MagicMock(return_value=()) router.get_configured_max_input_tokens = MagicMock(return_value=()) diff --git a/tests/unit/proxy/router_entry_point_stubs.py b/tests/unit/proxy/router_entry_point_stubs.py index b150faf9c6a..1d30cb99b44 100644 --- a/tests/unit/proxy/router_entry_point_stubs.py +++ b/tests/unit/proxy/router_entry_point_stubs.py @@ -99,6 +99,8 @@ def route_entry_points_to_stubs(router: _RouterDouble) -> _RouterDouble: return router.is_recognized_model(name) if kind == "upstream": return router.get_routable_upstream_model(name, **_given(team_id=team_id)) + if kind == "upstream_provider": + return router.get_routable_upstream_provider(name, **_given(team_id=team_id)) if kind == "search_tool_support": return router.get_search_tool_support(name, **_given(team_id=team_id)) if kind == "from_id": diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 4f35efdfd6e..d1a8942c967 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -13950,6 +13950,27 @@ def test_upstream_model_and_service_tiers_are_read_off_the_deployments_the_team_ assert router.get_configured_service_tiers(model_name, team_id) == service_tiers +@pytest.mark.parametrize( + ("litellm_params", "provider"), + [ + ({"model": "us.openai.gpt-6.1-sol", "custom_llm_provider": "bedrock"}, "bedrock"), + ({"model": "bedrock_mantle/openai.gpt-6.1-sol"}, "bedrock_mantle"), + ({"model": "gpt-6.1-sol"}, None), + ], +) +def test_the_upstream_provider_is_the_one_the_routable_deployment_declares(litellm_params, provider): + router = litellm.Router( + model_list=[ + {"model_name": "sol", "litellm_params": litellm_params, "model_info": {"team_id": "team-1"}}, + {"model_name": "sol", "litellm_params": {"model": "azure/sol-deployment"}, "model_info": {"team_id": "t2"}}, + ] + ) + + assert router.registry().resolve("sol", kind="upstream_provider", team_id="team-1") == provider + assert router.registry().resolve("sol", kind="upstream_provider", team_id="t2") == "azure" + assert router.registry().resolve("not-a-real-model", kind="upstream_provider") is None + + def test_get_configured_display_name_treats_malformed_values_as_absent(): malformed = ["", " ", 12345, ["Kimi K3"], {"name": "Kimi K3"}, True] router = litellm.Router( diff --git a/tests/unit/test_router/test_router_entry_points.py b/tests/unit/test_router/test_router_entry_points.py index 81c1982b987..b46ea5a5c88 100644 --- a/tests/unit/test_router/test_router_entry_points.py +++ b/tests/unit/test_router/test_router_entry_points.py @@ -656,6 +656,11 @@ FORWARDS = [ "_core_get_routable_upstream_model", (("gpt",), {"team_id": "t"}), ), + ( + lambda r: r.registry().resolve("gpt", kind="upstream_provider", team_id="t"), + "_core_get_routable_upstream_provider", + (("gpt",), {"team_id": "t"}), + ), ( lambda r: r.registry().resolve("gpt", kind="search_tool_support", team_id="t"), "_core_get_search_tool_support",