diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index df8c46fb66c..ad651b45f48 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -8,8 +8,6 @@ from collections.abc import Callable, Mapping, Sequence from types import MappingProxyType from typing import Final -from pydantic import TypeAdapter - import litellm from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.credential_accessor import CredentialAccessor @@ -31,8 +29,6 @@ _CREDENTIAL_LITELLM_PARAM_FIELDS = set(CredentialLiteLLMParams.model_fields) _CREDENTIAL_LITELLM_PARAM_FIELDS = set(CredentialLiteLLMParams.model_fields) -_ROUTER_DEPLOYMENT: Final = TypeAdapter(object) - def _check_wildcard_routing(model: str) -> bool: """ @@ -553,10 +549,8 @@ def _litellm_model(deployment: Mapping[str, object]) -> object: return litellm_params.get("model") if is_str_keyed_mapping(litellm_params) else None -def _router_deployment(registry: RouterRegistry, model_id: object) -> Mapping[str, object] | None: - if not isinstance(model_id, str): - return None - deployment: Final = _ROUTER_DEPLOYMENT.validate_python(registry.deployment_dict(model_id)) +def _router_deployment(registry: RouterRegistry, model_id: str) -> Mapping[str, object] | None: + deployment: Final[object] = registry.deployment_dict(model_id) return deployment if is_str_keyed_mapping(deployment) else None @@ -582,22 +576,17 @@ def _deployment_routed_model(source: Mapping[str, object], listed_name: str) -> def wildcard_routed_model(row: Mapping[str, object], llm_router: Router | None) -> str | None: - if llm_router is None: + model_id: Final = _model_info_field(row, "id") + listed_name: Final = row.get("model_name") + if llm_router is None or not isinstance(model_id, str) or not isinstance(listed_name, str): return None registry: Final = llm_router.registry() - source: Final = _router_deployment(registry, _model_info_field(row, "id")) - listed_name: Final = row.get("model_name") - if source is None or not isinstance(listed_name, str): + source: Final = _router_deployment(registry, model_id) + if source is None or _litellm_model(row) == _litellm_model(source): return None - if _litellm_model(row) == _litellm_model(source) or listed_name in registry.names(kind="group_set"): + if model_id not in registry.names(listed_name, kind="route_candidates", team_id=_deployment_team_id(row)): return None - routed_model: Final = _deployment_routed_model(source, listed_name) - return ( - routed_model - if routed_model is not None - and routed_model in registry.names(listed_name, kind="wildcard", team_id=_deployment_team_id(row)) - else None - ) + return _deployment_routed_model(source, listed_name) def _get_wildcard_models( diff --git a/tests/unit/proxy/proxy_server/test_routes_model_info.py b/tests/unit/proxy/proxy_server/test_routes_model_info.py index 2738b0fde7e..0968ecb1d24 100644 --- a/tests/unit/proxy/proxy_server/test_routes_model_info.py +++ b/tests/unit/proxy/proxy_server/test_routes_model_info.py @@ -676,9 +676,10 @@ def test_model_info_never_prices_a_wildcard_row_as_a_route_its_calls_skip( monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None, ) -> None: - """``acme-lite`` has a deployment of its own and ``my-openai/gpt-acme-*`` is a narrower wildcard, - so calls to those names never reach the catch-all or ``my-openai/*``, which still list them. Such a - row may go unpriced, but any price a row shows must be the one its calls are billed at.""" + """``acme-lite`` has a deployment of its own, ``acme-pro`` is an alias of it, ``acme-max`` is a + routing group over it, and ``my-openai/gpt-acme-*`` is a narrower wildcard, so calls to those names + never reach the catch-all or ``my-openai/*``, which still list them. Such a row may go unpriced, but + any price a row shows must be the one its calls are billed at.""" from litellm.proxy.auth import model_checks catalog: Final = { @@ -688,6 +689,18 @@ def test_model_info_never_prices_a_wildcard_row_as_a_route_its_calls_skip( "litellm_provider": "fireworks_ai", "mode": "chat", }, + "fireworks_ai/accounts/fireworks/models/acme-pro": { + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 2.5e-07, + "litellm_provider": "fireworks_ai", + "mode": "chat", + }, + "fireworks_ai/accounts/fireworks/models/acme-max": { + "input_cost_per_token": 3.5e-07, + "output_cost_per_token": 4.5e-07, + "litellm_provider": "fireworks_ai", + "mode": "chat", + }, "deepseek/acme-lite": { "input_cost_per_token": 7e-07, "output_cost_per_token": 8e-07, @@ -707,7 +720,11 @@ def test_model_info_never_prices_a_wildcard_row_as_a_route_its_calls_skip( model_checks, "get_provider_models", lambda provider, litellm_params=None: { - "fireworks_ai": ["fireworks_ai/accounts/fireworks/models/acme-lite"], + "fireworks_ai": [ + "fireworks_ai/accounts/fireworks/models/acme-lite", + "fireworks_ai/accounts/fireworks/models/acme-pro", + "fireworks_ai/accounts/fireworks/models/acme-max", + ], "openai": ["gpt-acme-lite"], }.get(provider, []), ) @@ -733,7 +750,9 @@ def test_model_info_never_prices_a_wildcard_row_as_a_route_its_calls_skip( "litellm_params": {"model": "deepseek/acme-*", "api_key": "x"}, "model_info": {"id": "deepseek-wildcard"}, }, - ] + ], + model_group_alias={"acme-pro": "acme-lite"}, + routing_groups=[{"group_name": "acme-max", "models": ["acme-lite"], "routing_strategy": "simple-shuffle"}], ) monkeypatch.setattr(proxy_server, "llm_router", router) monkeypatch.setattr(proxy_server, "llm_model_list", router.get_model_list()) @@ -753,7 +772,10 @@ def test_model_info_never_prices_a_wildcard_row_as_a_route_its_calls_skip( } assert sorted(listed) == [ ("deepseek-acme-lite", "acme-lite"), + ("deepseek-acme-lite", "acme-pro"), ("fireworks-wildcard", "acme-lite"), + ("fireworks-wildcard", "acme-max"), + ("fireworks-wildcard", "acme-pro"), ("openai-wildcard", "my-openai/gpt-acme-lite"), ] assert listed[("deepseek-acme-lite", "acme-lite")] == (7e-07, 8e-07) @@ -775,6 +797,78 @@ def test_model_info_never_prices_a_wildcard_row_as_a_route_its_calls_skip( _invalidate_model_cost_lowercase_map() +def test_model_info_prices_a_team_wildcard_row_by_the_route_its_team_takes( + client: TestClient, + auth_as: Callable[[], AbstractContextManager[object]], + monkeypatch: pytest.MonkeyPatch, + local_model_cost_map: None, +) -> None: + """Team ``team-a`` owns ``team-fw/*`` over Fireworks and its own DeepSeek deployment published as + ``team-fw/acme-chat``, so the team's calls to that name skip the wildcard while ``team-fw/acme-lite`` + goes through it.""" + from litellm.proxy.auth import model_checks + + catalog: Final = { + "fireworks_ai/accounts/fireworks/models/acme-chat": { + "input_cost_per_token": 1e-07, + "output_cost_per_token": 2e-07, + "litellm_provider": "fireworks_ai", + "mode": "chat", + }, + "fireworks_ai/accounts/fireworks/models/acme-lite": { + "input_cost_per_token": 5e-07, + "output_cost_per_token": 6e-07, + "litellm_provider": "fireworks_ai", + "mode": "chat", + }, + } + monkeypatch.setattr(litellm, "model_cost", {**litellm.model_cost, **catalog}) + _invalidate_model_cost_lowercase_map() + monkeypatch.setattr( + model_checks, + "get_provider_models", + lambda provider, litellm_params=None: { + "fireworks_ai": [ + "fireworks_ai/accounts/fireworks/models/acme-chat", + "fireworks_ai/accounts/fireworks/models/acme-lite", + ], + }.get(provider, []), + ) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "team-fw/*", + "litellm_params": {"model": "fireworks_ai/accounts/fireworks/models/*", "api_key": "x"}, + "model_info": {"id": "team-wildcard", "team_id": "team-a"}, + }, + { + "model_name": "model_name_team-a_acme-chat", + "litellm_params": {"model": "deepseek/acme-chat", "api_key": "x"}, + "model_info": { + "id": "team-deepseek", + "team_id": "team-a", + "team_public_model_name": "team-fw/acme-chat", + }, + }, + ] + ) + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "llm_model_list", router.get_model_list()) + monkeypatch.setattr(proxy_server, "user_model", None) + + with auth_as(): + response: Final = client.get("/model/info") + + assert response.status_code == 200, response.text + wildcard_rows: Final = { + row["model_name"]: row["model_info"].get("input_cost_per_token") + for row in response.json()["data"] + if row["model_info"]["id"] == "team-wildcard" and "*" not in row["model_name"] + } + assert wildcard_rows == {"team-fw/acme-chat": None, "team-fw/acme-lite": 5e-07} + _invalidate_model_cost_lowercase_map() + + # --------------------------------------------------------------------------- # GET /model/info — team BYOK scoping (issue #30983) # ---------------------------------------------------------------------------