mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-11 22:51:28 +00:00
fix(proxy): resolve /v1/models limits from the deployment, not the alias
This commit is contained in:
michelligabriele
parent
31ca4ddf32
commit
c60c60e6fe
9 changed files with 388 additions and 56 deletions
|
|
@ -7481,6 +7481,64 @@ async def get_available_models_for_user(
|
|||
return all_models
|
||||
|
||||
|
||||
def _safe_get_model_info(model: str, get_model_info: Callable[[str], ModelInfo]) -> ModelInfo | None:
|
||||
try:
|
||||
return get_model_info(model)
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.debug(
|
||||
"create_model_info_response: cost map lookup failed for %s: %s",
|
||||
model,
|
||||
e,
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _resolve_listing_model_info(
|
||||
deployment_model: str | None,
|
||||
listed_model: str,
|
||||
get_model_info: Callable[[str], ModelInfo],
|
||||
) -> tuple[ModelInfo, ...]:
|
||||
"""
|
||||
Cost-map entries describing a listed model, best source first.
|
||||
|
||||
The name a model is listed under is an arbitrary public alias, so it often misses the
|
||||
cost map and lands on a fallback-generalization rule that answers with a conservative
|
||||
family baseline instead of the real model's limits; the deployment's underlying model
|
||||
is what the request actually reaches. Both names are kept because either can
|
||||
generalize, and because a deployment's own model is registered into the cost map as a
|
||||
stub that carries no limits of its own. Exact entries are consulted before generalized
|
||||
ones, and each field is then taken from the first entry that has it.
|
||||
"""
|
||||
listed_info: Final = _safe_get_model_info(listed_model, get_model_info)
|
||||
|
||||
# Fast path, and the only one a wildcard-expanded name takes: with a single name
|
||||
# there is nothing to order, so skip the generalization test entirely. This keeps
|
||||
# the per-model cost of the listing on the hot path #33721 exists to protect.
|
||||
if deployment_model is None or deployment_model == listed_model:
|
||||
return () if listed_info is None else (listed_info,)
|
||||
|
||||
deployment_info: Final = _safe_get_model_info(deployment_model, get_model_info)
|
||||
if deployment_info is None:
|
||||
return () if listed_info is None else (listed_info,)
|
||||
if listed_info is None:
|
||||
return (deployment_info,)
|
||||
|
||||
from litellm.utils import is_generalized_model_info
|
||||
|
||||
# Both names resolved: the deployment's model leads unless it only generalized
|
||||
# while the listed name is an exact cost-map entry.
|
||||
if is_generalized_model_info(deployment_info) and not is_generalized_model_info(listed_info):
|
||||
return (listed_info, deployment_info)
|
||||
return (deployment_info, listed_info)
|
||||
|
||||
|
||||
def _first_token_limit(candidates: tuple[ModelInfo, ...], field: str) -> int | None:
|
||||
return next(
|
||||
(limit for limit in (coerce_token_limit(info.get(field)) for info in candidates) if limit is not None),
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
def create_model_info_response(
|
||||
model_id: str,
|
||||
provider: str,
|
||||
|
|
@ -7505,31 +7563,35 @@ def create_model_info_response(
|
|||
"owned_by": provider,
|
||||
}
|
||||
|
||||
try:
|
||||
model_cost_info: ModelInfo | None = get_model_info(model_id)
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.debug(
|
||||
"create_model_info_response: cost map lookup failed for %s: %s",
|
||||
model_id,
|
||||
e,
|
||||
)
|
||||
model_cost_info = None
|
||||
listing_info: Final = llm_router.get_model_listing_info(model_id) if llm_router is not None else None
|
||||
|
||||
max_input_tokens: int | None = None
|
||||
max_output_tokens: int | None = None
|
||||
if model_cost_info is not None:
|
||||
max_input_tokens = coerce_token_limit(model_cost_info.get("max_input_tokens"))
|
||||
max_output_tokens = coerce_token_limit(model_cost_info.get("max_output_tokens"))
|
||||
mode: Final = model_cost_info.get("mode")
|
||||
if isinstance(mode, str):
|
||||
base["mode"] = mode
|
||||
candidates: Final = _resolve_listing_model_info(
|
||||
deployment_model=listing_info.cost_map_key if listing_info is not None else None,
|
||||
listed_model=model_id,
|
||||
get_model_info=get_model_info,
|
||||
)
|
||||
|
||||
if llm_router is not None:
|
||||
configured_input, configured_output = llm_router.get_configured_token_limits(model_id)
|
||||
if configured_input is not None:
|
||||
max_input_tokens = configured_input
|
||||
if configured_output is not None:
|
||||
max_output_tokens = configured_output
|
||||
max_input_tokens: int | None = _first_token_limit(candidates, "max_input_tokens")
|
||||
max_output_tokens: int | None = _first_token_limit(candidates, "max_output_tokens")
|
||||
mode: Final = next(
|
||||
(
|
||||
m
|
||||
for m in (
|
||||
cast("Mapping[str, object]", info).get("mode") # cast-ok: an entry need not carry "mode"
|
||||
for info in candidates
|
||||
)
|
||||
if isinstance(m, str)
|
||||
),
|
||||
None,
|
||||
)
|
||||
if mode is not None:
|
||||
base["mode"] = mode
|
||||
|
||||
if listing_info is not None:
|
||||
if listing_info.max_input_tokens is not None:
|
||||
max_input_tokens = listing_info.max_input_tokens
|
||||
if listing_info.max_output_tokens is not None:
|
||||
max_output_tokens = listing_info.max_output_tokens
|
||||
|
||||
if max_input_tokens is not None:
|
||||
base["max_input_tokens"] = max_input_tokens
|
||||
|
|
|
|||
|
|
@ -200,6 +200,7 @@ from litellm.types.router import (
|
|||
CredentialLiteLLMParams,
|
||||
CustomRoutingStrategyBase,
|
||||
Deployment,
|
||||
DeploymentModelListingInfo,
|
||||
DeploymentTypedDict,
|
||||
FallbackAccessCheck,
|
||||
GuardrailTypedDict,
|
||||
|
|
@ -9726,25 +9727,44 @@ class Router:
|
|||
return None
|
||||
return Deployment(**first_usable) if isinstance(first_usable, dict) else first_usable
|
||||
|
||||
def get_model_listing_info(self, model_name: str) -> DeploymentModelListingInfo | None:
|
||||
"""
|
||||
Return what the concrete deployment behind model_name contributes to its
|
||||
/v1/models entry: the cost-map key for its underlying model, plus any token
|
||||
limits explicitly configured in its model_info. Resolved via O(1) index lookup.
|
||||
|
||||
Returns None for wildcard-expanded or unknown names, where the listed name is
|
||||
the real model name and no deployment-specific information exists, and treats a
|
||||
malformed configured limit as absent rather than failing the listing. Unlike
|
||||
get_model_group_info, this never triggers pattern matching or deep copies, so it
|
||||
is safe to call per listed model on the /v1/models hot path.
|
||||
"""
|
||||
deployment: Final = self.get_deployment_by_model_group_name(model_group_name=model_name)
|
||||
if deployment is None:
|
||||
return None
|
||||
|
||||
model_info: Final = deployment.model_info
|
||||
# base_model is a declared field, so read it as one: an unset or blank value
|
||||
# means the deployment's own model name is the cost-map key.
|
||||
base_model: Final = model_info.base_model
|
||||
return DeploymentModelListingInfo(
|
||||
cost_map_key=base_model or deployment.litellm_params.model,
|
||||
max_input_tokens=coerce_token_limit(model_info.get("max_input_tokens")),
|
||||
max_output_tokens=coerce_token_limit(model_info.get("max_output_tokens")),
|
||||
)
|
||||
|
||||
def get_configured_token_limits(self, model_name: str) -> "tuple[int | None, int | None]":
|
||||
"""
|
||||
Return (max_input_tokens, max_output_tokens) explicitly configured in a concrete
|
||||
deployment's model_info for model_name, via O(1) index lookup.
|
||||
|
||||
Returns (None, None) for wildcard-expanded or unknown names, and treats a
|
||||
malformed configured value as absent rather than failing the listing. Unlike
|
||||
get_model_group_info, this never triggers pattern matching or deep copies, so it
|
||||
is safe to call per listed model on the /v1/models hot path.
|
||||
malformed configured value as absent rather than failing the caller.
|
||||
"""
|
||||
deployment: Final = self.get_deployment_by_model_group_name(model_group_name=model_name)
|
||||
if deployment is None:
|
||||
listing_info: Final = self.get_model_listing_info(model_name=model_name)
|
||||
if listing_info is None:
|
||||
return (None, None)
|
||||
|
||||
model_info: Final = deployment.model_info
|
||||
return (
|
||||
coerce_token_limit(model_info.get("max_input_tokens")),
|
||||
coerce_token_limit(model_info.get("max_output_tokens")),
|
||||
)
|
||||
return (listing_info.max_input_tokens, listing_info.max_output_tokens)
|
||||
|
||||
def get_deployment_credentials_with_provider(
|
||||
self, model_id: str, team_id: str | None = None
|
||||
|
|
|
|||
|
|
@ -575,6 +575,23 @@ class Deployment(BaseModel):
|
|||
setattr(self, key, value)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DeploymentModelListingInfo:
|
||||
"""What a concrete deployment contributes to its OpenAI-compatible listing entry.
|
||||
|
||||
``cost_map_key`` is the name the deployment's underlying model is known by in
|
||||
``litellm.model_cost`` (``model_info.base_model`` when set, else
|
||||
``litellm_params.model``), which is what the request actually reaches; the public
|
||||
model name it is listed under is an arbitrary alias and often absent from the cost
|
||||
map. The token limits are the ones explicitly set in ``model_info``, which outrank
|
||||
anything the cost map says.
|
||||
"""
|
||||
|
||||
cost_map_key: str
|
||||
max_input_tokens: int | None
|
||||
max_output_tokens: int | None
|
||||
|
||||
|
||||
class RouterErrors(enum.Enum):
|
||||
"""
|
||||
Enum for router specific errors with common codes
|
||||
|
|
|
|||
|
|
@ -2948,24 +2948,33 @@ def _resolve_builtin_model_cost_entry(key: str, provider: str) -> dict[str, obje
|
|||
return None
|
||||
|
||||
|
||||
def is_generalized_model_info(model_info: ModelInfo) -> bool:
|
||||
"""Whether ``model_info`` came from a fallback-generalization capability rule.
|
||||
|
||||
Detected as the resolved key missing ``litellm.model_cost`` while matching a
|
||||
capability rule. A rule-derived entry carries no pricing and only a conservative
|
||||
family-baseline context window, so callers holding a second candidate name should
|
||||
prefer an exact cost-map entry from that name over this one.
|
||||
"""
|
||||
key: Final = cast("Mapping[str, object]", model_info).get("key") # cast-ok: partial dicts may omit "key"
|
||||
if not isinstance(key, str):
|
||||
return False
|
||||
return key not in litellm.model_cost and match_capability_generalizations(key) is not None
|
||||
|
||||
|
||||
def _get_builtin_model_info_for_registration(model: str) -> ModelInfo | None:
|
||||
"""Resolve ``model`` to its built-in cost-map entry for registration merging.
|
||||
|
||||
Returns ``None`` when the lookup raises or when it resolved via a
|
||||
fallback-generalization capability rule, detected as the resolved key missing
|
||||
``litellm.model_cost`` while matching a capability rule. A rule-derived entry
|
||||
carries no pricing, so treating it as a hit would skip the built-in
|
||||
cache-pricing inheritance for prefix-mangled keys.
|
||||
fallback-generalization capability rule. A rule-derived entry carries no
|
||||
pricing, so treating it as a hit would skip the built-in cache-pricing
|
||||
inheritance for prefix-mangled keys.
|
||||
"""
|
||||
try:
|
||||
info: Final = get_model_info(model=model)
|
||||
except Exception:
|
||||
return None
|
||||
if info["key"] in litellm.model_cost:
|
||||
return info
|
||||
if match_capability_generalizations(info["key"]) is None:
|
||||
return info
|
||||
return None
|
||||
return None if is_generalized_model_info(info) else info
|
||||
|
||||
|
||||
_runtime_registered_model_cost: Final[dict[str, dict[str, object]]] = {} # mutable-ok: replayed on reload
|
||||
|
|
|
|||
|
|
@ -1944,7 +1944,7 @@ class TestModelInfoEndpoint:
|
|||
):
|
||||
mock_router.get_fully_blocked_model_names.return_value = set()
|
||||
mock_router.get_model_list.return_value = []
|
||||
mock_router.get_configured_token_limits.return_value = (None, None)
|
||||
mock_router.get_model_listing_info.return_value = None
|
||||
mock_router.get_deployment_by_model_group_name.return_value = Deployment(
|
||||
model_name="gpt-4",
|
||||
litellm_params=LiteLLM_Params(model="openai/gpt-4"),
|
||||
|
|
@ -2021,7 +2021,7 @@ class TestModelInfoEndpoint:
|
|||
):
|
||||
mock_router.get_fully_blocked_model_names.return_value = set()
|
||||
mock_router.get_model_list.return_value = []
|
||||
mock_router.get_configured_token_limits.return_value = (None, None)
|
||||
mock_router.get_model_listing_info.return_value = None
|
||||
mock_router.get_deployment_by_model_group_name.return_value = Deployment(
|
||||
model_name="team-model-1",
|
||||
litellm_params=LiteLLM_Params(model="custom/team-model-1"),
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ import litellm
|
|||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy import utils as proxy_utils
|
||||
from litellm.proxy.utils import create_model_info_response
|
||||
from litellm.types.router import DeploymentModelListingInfo
|
||||
|
||||
from .conftest import normalize # type: ignore[import-not-found]
|
||||
|
||||
|
|
@ -165,13 +166,16 @@ def test_anthropic_format_carries_router_configured_token_limits(client, auth_as
|
|||
built from another entry's lookup shows up as the wrong numbers."""
|
||||
|
||||
def _configured(model_name):
|
||||
return (300000, 32000) if model_name == "gpt-4" else (500000, 4096)
|
||||
max_input, max_output = (300000, 32000) if model_name == "gpt-4" else (500000, 4096)
|
||||
return DeploymentModelListingInfo(
|
||||
cost_map_key=model_name, max_input_tokens=max_input, max_output_tokens=max_output
|
||||
)
|
||||
|
||||
def _cost_map_lookup(model_id):
|
||||
max_input, max_output = (200000, 64000) if model_id == "gpt-4" else (100000, 8000)
|
||||
return {"max_input_tokens": max_input, "max_output_tokens": max_output, "mode": "chat"}
|
||||
|
||||
patched_models.get_configured_token_limits = MagicMock(side_effect=_configured)
|
||||
patched_models.get_model_listing_info = MagicMock(side_effect=_configured)
|
||||
|
||||
def _resolved(**kwargs):
|
||||
return create_model_info_response(**kwargs, get_model_info=_cost_map_lookup)
|
||||
|
|
|
|||
|
|
@ -877,7 +877,7 @@ async def test_v1_models_translates_team_model_for_access_group_key(monkeypatch)
|
|||
router.get_model_names.return_value = ["model_name_teamX_uuid9"]
|
||||
router.get_model_access_groups.return_value = {"grp-a": ["model_name_teamX_uuid9"]}
|
||||
router.get_fully_blocked_model_names.return_value = set()
|
||||
router.get_configured_token_limits.return_value = (None, None)
|
||||
router.get_model_listing_info.return_value = None
|
||||
router.model_list = [team_dep]
|
||||
router.get_model_list.return_value = [team_dep]
|
||||
|
||||
|
|
@ -919,7 +919,7 @@ async def test_v1_models_keeps_internal_names_when_public_name_flag_disabled(
|
|||
router.get_model_names.return_value = ["model_name_teamX_uuid9"]
|
||||
router.get_model_access_groups.return_value = {"grp-a": ["model_name_teamX_uuid9"]}
|
||||
router.get_fully_blocked_model_names.return_value = set()
|
||||
router.get_configured_token_limits.return_value = (None, None)
|
||||
router.get_model_listing_info.return_value = None
|
||||
router.model_list = [team_dep]
|
||||
router.get_model_list.return_value = [team_dep]
|
||||
|
||||
|
|
@ -954,7 +954,7 @@ async def test_v1_models_translates_team_model_with_metadata(monkeypatch):
|
|||
router.get_model_names.return_value = ["model_name_teamX_uuid9"]
|
||||
router.get_model_access_groups.return_value = {"grp-a": ["model_name_teamX_uuid9"]}
|
||||
router.get_fully_blocked_model_names.return_value = set()
|
||||
router.get_configured_token_limits.return_value = (None, None)
|
||||
router.get_model_listing_info.return_value = None
|
||||
router.model_list = [team_dep]
|
||||
router.get_model_list.return_value = [team_dep]
|
||||
router.get_model_group_info.return_value = None
|
||||
|
|
@ -1000,7 +1000,7 @@ async def test_v1_models_metadata_fallbacks_use_internal_routing_key(monkeypatch
|
|||
router.get_model_names.return_value = ["model_name_teamX_uuid9"]
|
||||
router.get_model_access_groups.return_value = {"grp-a": ["model_name_teamX_uuid9"]}
|
||||
router.get_fully_blocked_model_names.return_value = set()
|
||||
router.get_configured_token_limits.return_value = (None, None)
|
||||
router.get_model_listing_info.return_value = None
|
||||
router.model_list = [team_dep]
|
||||
router.get_model_list.return_value = [team_dep]
|
||||
# Fallbacks are keyed on the internal routing name, as the router stores them.
|
||||
|
|
@ -1057,7 +1057,7 @@ async def test_v1_models_metadata_does_not_leak_other_team_fallbacks(monkeypatch
|
|||
router.get_model_names.return_value = ["model_name_teamX_uuid9"]
|
||||
router.get_model_access_groups.return_value = {"grp-a": ["model_name_teamX_uuid9"]}
|
||||
router.get_fully_blocked_model_names.return_value = set()
|
||||
router.get_configured_token_limits.return_value = (None, None)
|
||||
router.get_model_listing_info.return_value = None
|
||||
router.model_list = [team_x, team_y]
|
||||
router.get_model_list.return_value = [team_x, team_y]
|
||||
router.fallbacks = [
|
||||
|
|
@ -1312,7 +1312,7 @@ def test_translate_team_model_names_for_listing_respects_legacy_flag():
|
|||
def _public_named_router(*team_rows: dict) -> MagicMock:
|
||||
router = MagicMock()
|
||||
router.get_model_list.return_value = list(team_rows)
|
||||
router.get_configured_token_limits.return_value = (None, None)
|
||||
router.get_model_listing_info.return_value = None
|
||||
return router
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -886,6 +886,7 @@ from typing import cast
|
|||
|
||||
import litellm
|
||||
from litellm.proxy.utils import create_model_info_response
|
||||
from litellm.types.router import DeploymentModelListingInfo
|
||||
from litellm.types.utils import ModelInfo
|
||||
|
||||
|
||||
|
|
@ -913,7 +914,7 @@ def test_create_model_info_response_includes_max_tokens_from_lookup():
|
|||
|
||||
def test_create_model_info_response_does_not_call_router_group_info():
|
||||
router = MagicMock()
|
||||
router.get_configured_token_limits.return_value = (None, None)
|
||||
router.get_model_listing_info.return_value = None
|
||||
|
||||
response = create_model_info_response(
|
||||
model_id="some-model",
|
||||
|
|
@ -928,7 +929,9 @@ def test_create_model_info_response_does_not_call_router_group_info():
|
|||
|
||||
def test_create_model_info_response_uses_deployment_limits_when_not_in_cost_map():
|
||||
router = MagicMock()
|
||||
router.get_configured_token_limits.return_value = (32000, 8000)
|
||||
router.get_model_listing_info.return_value = DeploymentModelListingInfo(
|
||||
cost_map_key="my-custom-deployment", max_input_tokens=32000, max_output_tokens=8000
|
||||
)
|
||||
|
||||
response = create_model_info_response(
|
||||
model_id="my-custom-deployment",
|
||||
|
|
@ -944,7 +947,9 @@ def test_create_model_info_response_uses_deployment_limits_when_not_in_cost_map(
|
|||
|
||||
def test_create_model_info_response_deployment_limits_override_cost_map():
|
||||
router = MagicMock()
|
||||
router.get_configured_token_limits.return_value = (200000, None)
|
||||
router.get_model_listing_info.return_value = DeploymentModelListingInfo(
|
||||
cost_map_key="gpt-4o", max_input_tokens=200000, max_output_tokens=None
|
||||
)
|
||||
|
||||
response = create_model_info_response(
|
||||
model_id="gpt-4o",
|
||||
|
|
@ -1878,3 +1883,119 @@ async def test_proxy_only_error_5xx_keeps_traceback_and_runs_sync_callbacks(monk
|
|||
Logging.failure_handler = orig_sync_failure
|
||||
|
||||
assert "test_proxy_utils" in captured["async_traceback"]
|
||||
|
||||
|
||||
def test_create_model_info_response_resolves_alias_to_deployment_model():
|
||||
"""A public model name that is not itself a cost-map key must not be resolved through
|
||||
the fallback-generalization rules: `bedrock-claude-opus-5` matches the generic
|
||||
claude-family baseline (200k/64k) by substring, while the deployment it fronts really
|
||||
accepts 1M/128k. Regression for the /v1/models alias resolution introduced in v1.94.0."""
|
||||
from litellm import Router
|
||||
|
||||
saved_model_cost = dict(litellm.model_cost)
|
||||
try:
|
||||
router = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "bedrock-claude-opus-5",
|
||||
"litellm_params": {
|
||||
"custom_llm_provider": "bedrock",
|
||||
"model": "bedrock/eu.anthropic.claude-opus-5",
|
||||
},
|
||||
"model_info": {"base_model": "eu.anthropic.claude-opus-5"},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
response = create_model_info_response(
|
||||
model_id="bedrock-claude-opus-5", provider="openai", llm_router=router
|
||||
)
|
||||
finally:
|
||||
litellm.model_cost.clear()
|
||||
litellm.model_cost.update(saved_model_cost)
|
||||
|
||||
assert response["max_input_tokens"] == 1000000
|
||||
assert response["max_output_tokens"] == 128000
|
||||
|
||||
|
||||
def test_create_model_info_response_keeps_exact_alias_over_generalized_deployment_model():
|
||||
"""Mirror of the alias bug: when the deployment points at a custom backend name that
|
||||
only matches a generalization rule, the listed name's exact cost-map entry is the
|
||||
better answer and must win."""
|
||||
from litellm import Router
|
||||
|
||||
saved_model_cost = dict(litellm.model_cost)
|
||||
try:
|
||||
router = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "claude-opus-5",
|
||||
"litellm_params": {
|
||||
"custom_llm_provider": "bedrock",
|
||||
"model": "bedrock/my-claude-opus-5-provisioned",
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
response = create_model_info_response(
|
||||
model_id="claude-opus-5", provider="openai", llm_router=router
|
||||
)
|
||||
finally:
|
||||
litellm.model_cost.clear()
|
||||
litellm.model_cost.update(saved_model_cost)
|
||||
|
||||
assert response["max_input_tokens"] == 1000000
|
||||
|
||||
|
||||
def test_create_model_info_response_falls_back_to_alias_for_opaque_deployment_name():
|
||||
"""An Azure deployment named after the resource rather than the model has no cost-map
|
||||
entry; the listed name still does, and must keep answering."""
|
||||
from litellm import Router
|
||||
|
||||
saved_model_cost = dict(litellm.model_cost)
|
||||
try:
|
||||
router = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "gpt-4o",
|
||||
"litellm_params": {"model": "azure/my-gpt4o-deployment"},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
response = create_model_info_response(
|
||||
model_id="gpt-4o", provider="openai", llm_router=router
|
||||
)
|
||||
finally:
|
||||
litellm.model_cost.clear()
|
||||
litellm.model_cost.update(saved_model_cost)
|
||||
|
||||
assert response["max_input_tokens"] == 128000
|
||||
assert response["max_output_tokens"] == 16384
|
||||
|
||||
|
||||
def test_create_model_info_response_resolves_mode_through_deployment_model():
|
||||
"""`mode` is derived from the same lookup, so an aliased embedding deployment
|
||||
currently reports no mode at all; it must report `embedding`."""
|
||||
from litellm import Router
|
||||
|
||||
saved_model_cost = dict(litellm.model_cost)
|
||||
try:
|
||||
router = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "my-embeddings",
|
||||
"litellm_params": {"model": "openai/text-embedding-3-small"},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
response = create_model_info_response(
|
||||
model_id="my-embeddings", provider="openai", llm_router=router
|
||||
)
|
||||
finally:
|
||||
litellm.model_cost.clear()
|
||||
litellm.model_cost.update(saved_model_cost)
|
||||
|
||||
assert response["mode"] == "embedding"
|
||||
|
|
|
|||
|
|
@ -7271,6 +7271,105 @@ def test_get_configured_token_limits_coerces_numeric_strings():
|
|||
assert router.get_configured_token_limits("quoted-limits-model") == (32000, 8000)
|
||||
|
||||
|
||||
def test_get_model_listing_info_prefers_base_model_over_litellm_params_model():
|
||||
"""The cost-map key comes from base_model when set, so a deployment pointing at an
|
||||
opaque backend name still resolves the real catalog entry."""
|
||||
router = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "bedrock-claude-opus-5",
|
||||
"litellm_params": {"model": "bedrock/eu.anthropic.claude-opus-5"},
|
||||
"model_info": {"base_model": "eu.anthropic.claude-opus-5"},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
info = router.get_model_listing_info("bedrock-claude-opus-5")
|
||||
assert info is not None
|
||||
assert info.cost_map_key == "eu.anthropic.claude-opus-5"
|
||||
|
||||
|
||||
def test_get_model_listing_info_falls_back_to_litellm_params_model():
|
||||
router = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "bedrock-claude-opus-5",
|
||||
"litellm_params": {"model": "bedrock/eu.anthropic.claude-opus-5"},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
info = router.get_model_listing_info("bedrock-claude-opus-5")
|
||||
assert info is not None
|
||||
assert info.cost_map_key == "bedrock/eu.anthropic.claude-opus-5"
|
||||
|
||||
|
||||
def test_get_model_listing_info_ignores_blank_base_model():
|
||||
"""A base_model set to an empty string is absent, not a cost-map key."""
|
||||
router = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "bedrock-claude-opus-5",
|
||||
"litellm_params": {"model": "bedrock/eu.anthropic.claude-opus-5"},
|
||||
"model_info": {"base_model": ""},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
info = router.get_model_listing_info("bedrock-claude-opus-5")
|
||||
assert info is not None
|
||||
assert info.cost_map_key == "bedrock/eu.anthropic.claude-opus-5"
|
||||
|
||||
|
||||
def test_get_model_listing_info_returns_none_for_unknown_name():
|
||||
router = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "no-limits-model",
|
||||
"litellm_params": {"model": "openai/some-unmapped-model"},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
assert router.get_model_listing_info("not-a-real-model") is None
|
||||
|
||||
|
||||
def test_get_model_listing_info_carries_configured_limits():
|
||||
router = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "my-custom-model",
|
||||
"litellm_params": {"model": "openai/some-unmapped-model"},
|
||||
"model_info": {"max_input_tokens": 32000, "max_output_tokens": 8000},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
info = router.get_model_listing_info("my-custom-model")
|
||||
assert info is not None
|
||||
assert (info.max_input_tokens, info.max_output_tokens) == (32000, 8000)
|
||||
|
||||
|
||||
def test_get_model_listing_info_skips_wildcard_pattern_matching():
|
||||
router = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "bedrock/*",
|
||||
"litellm_params": {"model": "bedrock/*"},
|
||||
"model_info": {"max_input_tokens": 12345},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
with patch.object(
|
||||
router.pattern_router, "route", side_effect=AssertionError("pattern route called")
|
||||
):
|
||||
assert (
|
||||
router.get_model_listing_info("bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0")
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_acreate_batch_disable_fallbacks_surfaces_owning_provider_error():
|
||||
router = litellm.Router(
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue