diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index c1db8b7b4c7..7f2aa33246d 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -7,21 +7,27 @@ import functools import itertools import re from collections.abc import Callable, Collection, Mapping, Sequence +from collections.abc import Set as AbstractSet from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass, replace from types import MappingProxyType from typing import Final, TypeVar import litellm from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.credential_accessor import CredentialAccessor +from litellm.litellm_core_utils.litellm_logging import use_custom_pricing_for_model from litellm.litellm_core_utils.pattern_match_deployments import PatternMatchRouter from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping from litellm.proxy._types import SpecialModelNames, UserAPIKeyAuth +from litellm.proxy.common_utils.model_listing_utils import CallerAliases, alias_map, dispatched_model +from litellm.proxy.common_utils.token_count_settings import cost_map_entry from litellm.repositories.object_permission_repository import ObjectPermissionRepository from litellm.responses.main import strip_responses_routing_prefix from litellm.router import Router -from litellm.router_api import fallback_chain -from litellm.types.router import CredentialLiteLLMParams, LiteLLM_Params +from litellm.router_api import RouterRegistry, fallback_chain +from litellm.router_rust.surface.registry import cost_map_backend_model +from litellm.types.router import CredentialLiteLLMParams, Deployment, LiteLLM_Params from litellm.types.utils import LlmProviders from litellm.utils import ( AZURE_MODEL_PLACEHOLDER, @@ -684,6 +690,133 @@ def _known_models_without_a_deployment(wildcard_model: str) -> Sequence[str]: return get_known_models_from_wildcard(wildcard_model=wildcard_model, litellm_params=None) +def _litellm_model(deployment: Mapping[str, object]) -> object: + litellm_params: Final = deployment.get("litellm_params") + return litellm_params.get("model") if is_str_keyed_mapping(litellm_params) else None + + +def _router_deployment(registry: RouterRegistry, model_id: str) -> Mapping[str, object] | None: + deployment: Final[object] = registry.deployment_dict(model_id) + return deployment if is_str_keyed_mapping(deployment) else None + + +GLOBAL_ALIASES_ONLY: Final = CallerAliases(team=None, key=None) + + +@dataclass(frozen=True, slots=True) +class RoutedElsewhere: + pass + + +ROUTED_ELSEWHERE: Final = RoutedElsewhere() + + +@dataclass(frozen=True, slots=True) +class OtherDeploymentsBill: + model: str + + +def wildcard_row_priced_model( + row: Mapping[str, object], llm_router: Router | None, caller_aliases: CallerAliases = GLOBAL_ALIASES_ONLY +) -> str | OtherDeploymentsBill | RoutedElsewhere | None: + model_id: Final = _model_info_field(row, "id") + listed_name: Final = row.get("model_name") + if llm_router is None or not isinstance(model_id, str) or not isinstance(listed_name, str): + return None + registry: Final = llm_router.registry() + source: Final = _router_deployment(registry, model_id) + if source is None: + return None + source_pattern: Final = source.get("model_name") + source_model: Final = _litellm_model(source) + if not isinstance(source_pattern, str) or not isinstance(source_model, str) or _litellm_model(row) == source_model: + return None + return wildcard_deployment_priced_model( + llm_router, + WildcardDeployment(model_id, source_pattern, source_model, _deployment_team_id(row)), + listed_name, + caller_aliases, + ) + + +@dataclass(frozen=True, slots=True) +class WildcardDeployment: + model_id: str + model_name_pattern: str + litellm_model: str + team_id: str | None + + +def wildcard_deployment_priced_model( + llm_router: Router, deployment: WildcardDeployment, listed_name: str, caller_aliases: CallerAliases +) -> str | OtherDeploymentsBill | RoutedElsewhere | None: + served_name: Final = _served_name(llm_router, listed_name, caller_aliases) + registry: Final = llm_router.registry() + team_id: Final = deployment.team_id or caller_aliases.team_id + routed_name: Final = routed_model_name(llm_router, served_name) + candidates: Final = registry.names(served_name, kind="route_candidates", team_id=team_id) + if deployment.model_id in candidates: + return _routed_deployment_model(deployment.model_name_pattern, routed_name, deployment.litellm_model) + if not candidates: + return None + return _model_other_deployments_bill(registry, routed_name, candidates, team_id) + + +def _model_other_deployments_bill( + registry: RouterRegistry, routed_name: str, candidates: AbstractSet[str], team_id: str | None +) -> OtherDeploymentsBill | RoutedElsewhere: + rows: Final = registry.deployments(routed_name, team_id=team_id) or () + billed: Final = frozenset( + cost_map_backend_model(Deployment.model_validate(row)) + for row in rows + if _model_info_field(row, "id") in candidates + ) + if len(billed) != 1 or any(_declares_its_own_rate(registry, model_id) for model_id in candidates): + return ROUTED_ELSEWHERE + return OtherDeploymentsBill(next(iter(billed))) + + +def _declares_its_own_rate(registry: RouterRegistry, model_id: str) -> bool: + deployment: Final = _router_deployment(registry, model_id) + if deployment is None: + return False + sections: Final = (deployment.get("litellm_params"), deployment.get("model_info")) + return any(is_str_keyed_mapping(section) and use_custom_pricing_for_model(dict(section)) for section in sections) + + +def wildcard_row_filled_model( + llm_router: Router, name_pattern: str, litellm_model: str, listed_name: str, caller_aliases: CallerAliases +) -> str: + served_name: Final = _served_name(llm_router, listed_name, caller_aliases) + return _routed_deployment_model(name_pattern, routed_model_name(llm_router, served_name), litellm_model) + + +def _served_name(llm_router: Router, listed_name: str, caller_aliases: CallerAliases) -> str: + return dispatched_model(listed_name, _without_deleted_team_targets(caller_aliases, llm_router.registry())) + + +def _without_deleted_team_targets(caller_aliases: CallerAliases, registry: RouterRegistry) -> CallerAliases: + deleted_prefix: Final = f"model_name_{caller_aliases.team_id}_" + live_names: Final = registry.index("model_name") + return replace( + caller_aliases, + team={ + alias: target + for alias, target in alias_map(caller_aliases.team).items() + if not target.startswith(deleted_prefix) or target in live_names + }, + ) + + +def routed_model_name(llm_router: Router, listed_name: str) -> str: + return llm_router.registry().resolve(listed_name, kind="alias") or listed_name + + +def model_info_cost_map_entry(model: str) -> Mapping[str, object]: + provider, _, _ = model.partition("/") + return cost_map_entry(model) or cost_map_entry(model.rsplit("/", 1)[-1], custom_llm_provider=provider) + + def _get_wildcard_models( unique_models: list[str], return_wildcard_routes: bool | None = False, diff --git a/litellm/proxy/common_utils/model_listing_utils.py b/litellm/proxy/common_utils/model_listing_utils.py index 9db67aa3808..86b444663f0 100644 --- a/litellm/proxy/common_utils/model_listing_utils.py +++ b/litellm/proxy/common_utils/model_listing_utils.py @@ -160,8 +160,8 @@ class ClaudeCodeRoutingNames: @dataclass(frozen=True, slots=True) class CallerAliases: - """The caller's `team` and `key` alias maps, the names `/v1/models` lists for it, and `router`, - its key's or team's `router_settings.model_group_alias`. + """The caller's `team` and `key` alias maps, the names `/v1/models` lists for it, `router`, + its key's or team's `router_settings.model_group_alias`, and `team_id`, the key's team. `rewrite` are the maps `/chat/completions` rewrites its model through after auth's `router` rewrite, in the order it applies them: the team's, the key's in `add_litellm_data_to_request`, then the global `model_alias_map` and the key's again in `common_processing_pre_call_logic`, @@ -170,6 +170,7 @@ class CallerAliases: team: object key: object router: object = None + team_id: str | None = None @property def own(self) -> tuple[object, ...]: @@ -214,7 +215,7 @@ def caller_alias_maps( """Team aliases count only when listing the team the key authenticated as.""" if listed_team_id is not None and listed_team_id != key_team_id: return CallerAliases(None, key_aliases, router_aliases) - return CallerAliases(team_aliases, key_aliases, router_aliases) + return CallerAliases(team_aliases, key_aliases, router_aliases, key_team_id) def alias_map(aliases: object) -> Mapping[str, str]: diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 3c8f7d6d64f..38b814225ce 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -15,7 +15,7 @@ import datetime import itertools import json from collections.abc import AsyncGenerator, Awaitable, Callable, Mapping, Sequence -from contextlib import AbstractAsyncContextManager, asynccontextmanager, suppress +from contextlib import AbstractAsyncContextManager, asynccontextmanager from dataclasses import dataclass from fnmatch import fnmatchcase from functools import partial @@ -62,6 +62,14 @@ from litellm.proxy._types import ( UserAPIKeyAuth, ) from litellm.proxy.auth.litellm_license import AUTO_ROUTER_LICENSE_REMEDY +from litellm.proxy.auth.model_checks import ( + GLOBAL_ALIASES_ONLY, + OtherDeploymentsBill, + WildcardDeployment, + model_info_cost_map_entry, + wildcard_deployment_priced_model, + wildcard_row_filled_model, +) from litellm.proxy.auth.team_grants import team_model_aliases from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.config_sync_pubsub import ( @@ -78,6 +86,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import ( decrypt_value_helper, encrypt_value_helper, ) +from litellm.proxy.common_utils.model_listing_utils import CallerAliases from litellm.proxy.common_utils.openai_error_payload import proxy_exception_for from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.proxy.db.routing_prisma_wrapper import WriterPinnedClient @@ -1314,14 +1323,47 @@ def _ptu_priced_deployment(model_params: Deployment) -> Deployment: ) -def _cost_map_entry(db_model: Deployment, incoming_model_info: Mapping[str, object]) -> Mapping[str, object]: +@dataclass(frozen=True, slots=True) +class _ListedRow: + name: str | None = None + llm_router: Router | None = None + caller_aliases: CallerAliases = GLOBAL_ALIASES_ONLY + + +_NO_LISTED_ROW: Final = _ListedRow() + + +def _live_cost_map_model(db_model: Deployment, listed_row: _ListedRow) -> str | None: + stored_model: Final = _decrypted_model(db_model.litellm_params.model) + if stored_model is None or "*" not in stored_model or listed_row.name is None or listed_row.llm_router is None: + return stored_model + return wildcard_row_filled_model( + listed_row.llm_router, db_model.model_name, stored_model, listed_row.name, listed_row.caller_aliases + ) + + +def _cost_map_entry( + db_model: Deployment, incoming_model_info: Mapping[str, object], listed_row: _ListedRow = _NO_LISTED_ROW +) -> Mapping[str, object]: base_model: Final = incoming_model_info.get("base_model") - lookup: Final = base_model if isinstance(base_model, str) else _decrypted_model(db_model.litellm_params.model) - if lookup is None: + lookup: Final = base_model if isinstance(base_model, str) else _live_cost_map_model(db_model, listed_row) + return MappingProxyType({}) if lookup is None else model_info_cost_map_entry(lookup) + + +def _routed_cost_map_entry(db_model: Deployment, listed_row: _ListedRow) -> Mapping[str, object]: + stored_model: Final = _decrypted_model(db_model.litellm_params.model) + if stored_model is None or "*" not in stored_model or listed_row.name is None or listed_row.llm_router is None: return MappingProxyType({}) - with suppress(Exception): - return MappingProxyType(dict(litellm.get_model_info(model=lookup))) - return MappingProxyType({}) + priced_as: Final = wildcard_deployment_priced_model( + listed_row.llm_router, + WildcardDeployment( + db_model.model_info.id or "", db_model.model_name, stored_model, db_model.model_info.team_id + ), + listed_row.name, + listed_row.caller_aliases, + ) + billed_model: Final = priced_as.model if isinstance(priced_as, OtherDeploymentsBill) else priced_as + return model_info_cost_map_entry(billed_model) if isinstance(billed_model, str) else MappingProxyType({}) LoadedCatalog: TypeAlias = Callable[[], Mapping[str, Mapping[str, object]]] @@ -1361,6 +1403,7 @@ def legacy_update_model_info_changes( incoming_model_info: ModelInfo, discovered_model_info: Mapping[str, object] = MappingProxyType({}), loaded_catalog: LoadedCatalog = GetModelCostMap.loaded_model_cost_map, + listed_row: _ListedRow = _NO_LISTED_ROW, ) -> Mapping[str, object]: """The ``model_info`` fields ``POST /model/update`` persists from its request. @@ -1380,7 +1423,8 @@ def legacy_update_model_info_changes( echoed: Final = frozenset( echoed_cost_map_fields( requested, - _cost_map_entry(db_model, {**stored, **sent}), + _cost_map_entry(db_model, {**stored, **sent}, listed_row), + _routed_cost_map_entry(db_model, listed_row), _loaded_catalog_entry(requested, loaded_catalog), discovered_model_info, ) @@ -1399,9 +1443,10 @@ def _legacy_update_model_info_writes( incoming_model_info: ModelInfo, discovered_model_info: Mapping[str, object], member_marker: bool | None, + listed_row: _ListedRow, ) -> Mapping[str, object]: changes: Final = { - **legacy_update_model_info_changes(current, incoming_model_info, discovered_model_info), + **legacy_update_model_info_changes(current, incoming_model_info, discovered_model_info, listed_row=listed_row), **({"member_auto_router": member_marker} if member_marker is not None else {}), } return MappingProxyType({k: v for k, v in changes.items() if getattr(current.model_info, k, None) != v}) @@ -1413,13 +1458,14 @@ async def _locked_legacy_update_model_info_json( incoming_model_info: ModelInfo, discovered_model_info: Mapping[str, object], member_marker: bool | None, + listed_row: _ListedRow, ) -> str | None: row: Final = await table.find_unique(where={"model_id": model_id}) if row is None: return None payload: Final = row.model_dump() writes: Final = _legacy_update_model_info_writes( - Deployment(**payload), incoming_model_info, discovered_model_info, member_marker + Deployment(**payload), incoming_model_info, discovered_model_info, member_marker, listed_row ) if not writes: return None @@ -1445,6 +1491,9 @@ def update_db_model( db_model: Deployment, updated_patch: updateDeployment, loaded_catalog: LoadedCatalog = GetModelCostMap.loaded_model_cost_map, + llm_router: Router | None = None, + caller_aliases: CallerAliases = GLOBAL_ALIASES_ONLY, + listed_name: str | None = None, ) -> PrismaCompatibleUpdateDBModel: if updated_patch.model_info is not None: _raise_if_ptu_cost_attribution_disabled(updated_patch.model_info.model_dump(exclude_none=True)) @@ -1474,9 +1523,11 @@ def update_db_model( # update model info if updated_patch.model_info: incoming_model_info: Final = updated_patch.model_info.model_dump(exclude_none=True) + listed_row: Final = _ListedRow(listed_name or updated_patch.model_name, llm_router, caller_aliases) echoed_fields: Final = echoed_cost_map_fields( incoming_model_info, - _cost_map_entry(db_model, incoming_model_info), + _cost_map_entry(db_model, incoming_model_info, listed_row), + _routed_cost_map_entry(db_model, listed_row), _loaded_catalog_entry(incoming_model_info, loaded_catalog), ) stored_model_info_fields: Final[Mapping[str, object]] = without_server_derived_pricing(incoming_model_info) @@ -1656,6 +1707,7 @@ async def patch_model( llm_router, premium_user, prisma_client, + request_caller_aliases, store_model_in_db, user_api_key_cache, ) @@ -1826,6 +1878,8 @@ async def patch_model( user_api_key_dict=user_api_key_dict, prisma_client=prisma_client, write_row=write_row, + llm_router=llm_router, + caller_aliases=await request_caller_aliases(user_api_key_dict), ) if updated_model is None: @@ -2181,6 +2235,8 @@ async def _update_team_model_in_db( user_api_key_dict: UserAPIKeyAuth, prisma_client: PrismaClient, write_row: Callable[[PrismaCompatibleUpdateDBModel], Awaitable[_RowT]], + llm_router: Router | None = None, + caller_aliases: CallerAliases = GLOBAL_ALIASES_ONLY, ) -> _RowT: """ Handle team model updates with proper alias management. @@ -2225,7 +2281,11 @@ async def _update_team_model_in_db( # No team_id in patch, proceed with standard update if patch_team_id is None: - return await write_row(update_db_model(db_model=db_model, updated_patch=patch_data)) + return await write_row( + update_db_model( + db_model=db_model, updated_patch=patch_data, llm_router=llm_router, caller_aliases=caller_aliases + ) + ) # Determine public model name public_model_name: Final = _get_public_model_name( @@ -2246,7 +2306,15 @@ async def _update_team_model_in_db( ), } ) - row: Final = await write_row(update_db_model(db_model=db_model, updated_patch=team_patch)) + row: Final = await write_row( + update_db_model( + db_model=db_model, + updated_patch=team_patch, + llm_router=llm_router, + caller_aliases=caller_aliases, + listed_name=patch_data.model_name, + ) + ) if is_new_team_assignment: await _setup_new_team_model_assignment( @@ -3292,6 +3360,7 @@ async def update_model( llm_router, premium_user, prisma_client, + request_caller_aliases, store_model_in_db, user_api_key_cache, ) @@ -3429,8 +3498,11 @@ async def update_model( if llm_router is not None else MappingProxyType({}) ) + listed_row: Final = _ListedRow( + model_params.model_name, llm_router, await request_caller_aliases(user_api_key_dict) + ) requested_model_info: Final = _legacy_update_model_info_writes( - deployment, _model_info, discovered_model_info, member_marker + deployment, _model_info, discovered_model_info, member_marker, listed_row ) _raise_if_access_groups_malformed(requested_model_info) async with _auto_router_capability_slot( @@ -3442,7 +3514,7 @@ async def update_model( ) as table: model_info_json: Final = ( await _locked_legacy_update_model_info_json( - table, _model_id, _model_info, discovered_model_info, member_marker + table, _model_id, _model_info, discovered_model_info, member_marker, listed_row ) if requested_model_info else None diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 8367a63f8c5..d21b6169133 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -471,13 +471,18 @@ from litellm.proxy.auth.master_key_boot_check import ( with_stored_secrets_counted, ) from litellm.proxy.auth.model_checks import ( + GLOBAL_ALIASES_ONLY, + OtherDeploymentsBill, + RoutedElsewhere, expand_wildcard_deployments_for_model_info, get_all_fallbacks, get_complete_model_list, get_key_models, get_mcp_server_ids, get_team_models, + model_info_cost_map_entry, run_model_listing, + wildcard_row_priced_model, ) from litellm.proxy.auth.password_policy import validate_password_not_breached, validate_password_policy from litellm.proxy.auth.user_api_key_auth import ( # noqa: F401, RUF100 # legacy module exports @@ -11108,7 +11113,7 @@ def _pricing_override_stamps( return MappingProxyType({**own_pricing, PRICING_OVERRIDES_KEY: pricing_override_fields(model_info, own_pricing)}) -def get_litellm_model_info(model: dict = {}): +def get_litellm_model_info(model: dict = {}) -> Mapping[str, object]: model_info: Final = model.get("model_info", {}) model_to_lookup = model.get("litellm_params", {}).get("model", None) try: @@ -12656,16 +12661,16 @@ async def _deployment_hidden_by_listing_callbacks(deployment: Deployment, user_a return listed_name in await _names_hidden_by_listing_callbacks(user_api_key_dict, (listed_name,)) -async def _caller_aliases(user_api_key_dict: UserAPIKeyAuth, team_id: str | None) -> CallerAliases: - caller_router_settings: Final = await proxy_config.get_hierarchical_router_settings( +async def request_caller_aliases(user_api_key_dict: UserAPIKeyAuth, listed_team_id: str | None = None) -> CallerAliases: + router_settings: Final = await proxy_config.get_hierarchical_router_settings( user_api_key_dict, prisma_client, proxy_logging_obj ) return caller_alias_maps( user_api_key_dict.aliases, user_api_key_dict.team_model_aliases, user_api_key_dict.team_id, - team_id, - caller_router_settings.get("model_group_alias") if isinstance(caller_router_settings, Mapping) else None, + listed_team_id, + router_settings.get("model_group_alias") if isinstance(router_settings, Mapping) else None, ) @@ -12758,7 +12763,7 @@ async def _key_model_listing( async def codex_catalog_rows_for_key(user_api_key_dict: UserAPIKeyAuth) -> tuple[CodexCatalogRow, ...]: listing: Final = await _key_model_listing( user_api_key_dict, - await _caller_aliases(user_api_key_dict, None), + await request_caller_aliases(user_api_key_dict), await _listing_hidden_names(None), ) return await asyncio.to_thread( @@ -12841,7 +12846,7 @@ async def model_list( http_request is not None and http_request.headers.get("anthropic-version") is not None ) client_headers: Final[Mapping[str, str]] = http_request.headers if http_request is not None else _EMPTY_HEADERS - caller_aliases: Final = await _caller_aliases(user_api_key_dict, team_id) + caller_aliases: Final = await request_caller_aliases(user_api_key_dict, team_id) routing_names: Final = ClaudeCodeRoutingNames( llm_router, team_id or user_api_key_dict.team_id, @@ -13085,16 +13090,7 @@ async def model_info( ) hidden_names: Final = blocked_names | unhealthy_names internal_to_public: Final = TeamModelNameTranslator.build_internal_to_public_map(llm_router, settings) - caller_router_settings: Final = await proxy_config.get_hierarchical_router_settings( - user_api_key_dict, prisma_client, proxy_logging_obj - ) - caller_aliases: Final = caller_alias_maps( - user_api_key_dict.aliases, - user_api_key_dict.team_model_aliases, - user_api_key_dict.team_id, - team_id, - caller_router_settings.get("model_group_alias") if isinstance(caller_router_settings, Mapping) else None, - ) + caller_aliases: Final = await request_caller_aliases(user_api_key_dict, team_id) callback_hidden_names: Final = await _names_hidden_by_listing_callbacks( user_api_key_dict, tuple( @@ -16808,7 +16804,10 @@ async def get_all_team_and_direct_access_models( def _enrich_model_info_with_litellm_data( - model: dict[str, Any], debug: bool = False, llm_router: Router | None = None + model: dict[str, Any], + debug: bool = False, + llm_router: Router | None = None, + caller_aliases: CallerAliases = GLOBAL_ALIASES_ONLY, ) -> dict[str, Any]: """ Enrich a model dictionary with litellm model info (pricing, context window, etc.) @@ -16818,6 +16817,7 @@ def _enrich_model_info_with_litellm_data( model: Model dictionary to enrich debug: Whether to include debug information like openai_client llm_router: Optional router instance for debug info + caller_aliases: The caller's alias maps, applied before deciding which model a wildcard row prices as Returns: Enriched model dictionary with sensitive info removed @@ -16835,30 +16835,29 @@ def _enrich_model_info_with_litellm_data( # read litellm model_prices_and_context_window.json to get the following: # input_cost_per_token, output_cost_per_token, max_tokens - litellm_model_info = get_litellm_model_info(model=model) - - # 2nd pass on the model, try seeing if we can find model in litellm model_cost map - if litellm_model_info == {}: - # use litellm_param model_name to get model_info - litellm_params = model.get("litellm_params", {}) - litellm_model = litellm_params.get("model", None) - try: - litellm_model_info = litellm.get_model_info(model=litellm_model) - except Exception: - litellm_model_info = {} - # 3rd pass on the model, try seeing if we can find model but without the "/" in model cost map - if litellm_model_info == {}: - # use litellm_param model_name to get model_info - litellm_params = model.get("litellm_params", {}) - litellm_model = litellm_params.get("model", None) - if litellm_model: - split_model: Final = litellm_model.split("/") - if len(split_model) > 0: - litellm_model = split_model[-1] - try: - litellm_model_info = litellm.get_model_info(model=litellm_model, custom_llm_provider=split_model[0]) - except Exception: - litellm_model_info = {} + priced_as: Final = wildcard_row_priced_model(model, llm_router, caller_aliases) + billed_model: Final = priced_as.model if isinstance(priced_as, OtherDeploymentsBill) else priced_as + lookup_model_info: Final = ( + {k: v for k, v in model_info.items() if k != "base_model"} + if isinstance(priced_as, OtherDeploymentsBill) + else model_info + ) + priced_model: Final = ( + {**model, "litellm_params": {**model["litellm_params"], "model": billed_model}, "model_info": lookup_model_info} + if isinstance(billed_model, str) + else model + ) + priced_litellm_model: Final = priced_model.get("litellm_params", {}).get("model") + litellm_model_info: Final[Mapping[str, object]] = ( + MappingProxyType({}) + if isinstance(priced_as, RoutedElsewhere) + else get_litellm_model_info(model=priced_model) + or ( + model_info_cost_map_entry(priced_litellm_model) + if isinstance(priced_litellm_model, str) + else MappingProxyType({}) + ) + ) discovered_model_info: Final = ( llm_router.registry().discovered_model_info(model_info.get("id")) if llm_router is not None @@ -18475,8 +18474,10 @@ def _translate_model_name_for_response(model: dict) -> dict[str, object]: return {**model, "model_name": team_public} -def _get_proxy_model_info(model: dict) -> dict: - return _translate_model_name_for_response(_enrich_model_info_with_litellm_data(model=model, llm_router=llm_router)) +def _get_proxy_model_info(model: dict, caller_aliases: CallerAliases = GLOBAL_ALIASES_ONLY) -> dict: + return _translate_model_name_for_response( + _enrich_model_info_with_litellm_data(model=model, llm_router=llm_router, caller_aliases=caller_aliases) + ) def _model_info_json_response(data: Sequence[Mapping[str, object]] | Mapping[str, object]) -> Response: @@ -18623,7 +18624,10 @@ async def model_info_v1( status_code=400, detail={"error": f"Model id = {litellm_model_id} not found on litellm proxy"}, ) - _deployment_info_dict = _get_proxy_model_info(model=deployment_info.model_dump(exclude_none=True)) + _deployment_info_dict = _get_proxy_model_info( + model=deployment_info.model_dump(exclude_none=True), + caller_aliases=await request_caller_aliases(user_api_key_dict), + ) single_model_list: list[dict] = [_deployment_info_dict] if prisma_client is not None: single_model_list = await _populate_team_access_on_models( @@ -18699,8 +18703,11 @@ async def model_info_v1( if include_team_models: all_models = _filter_models_to_user_accessible(all_models) + caller_aliases: Final = await request_caller_aliases(user_api_key_dict) all_models = [ - _translate_model_name_for_response(_enrich_model_info_with_litellm_data(model=model, llm_router=llm_router)) + _translate_model_name_for_response( + _enrich_model_info_with_litellm_data(model=model, llm_router=llm_router, caller_aliases=caller_aliases) + ) for model in all_models ] diff --git a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py index ee16b68b750..2b269469f89 100644 --- a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py @@ -4869,6 +4869,146 @@ class TestModelInfoCostMapEchoFilter: "cost-map metadata must not be persisted from an unchanged /model/info echo" ) + def test_echo_of_an_expanded_wildcard_row_is_not_persisted_on_the_wildcard(self): + import litellm + from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model + from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo + + router: Final = litellm.Router( + model_list=[ + { + "model_name": "my-openai/*", + "litellm_params": {"model": "openai/*", "api_key": "x"}, + "model_info": {"id": "dep-echo-wildcard"}, + } + ] + ) + entry: Final = litellm.get_model_info("openai/gpt-5.6") + echo: Final = {**entry, "id": "dep-echo-wildcard", "db_model": True, "access_groups": ["prod"]} + db_model: Final = Deployment( + model_name="my-openai/*", + litellm_params=LiteLLM_Params(model="openai/*"), + model_info=ModelInfo(id="dep-echo-wildcard"), + ) + + result: Final = update_db_model( + db_model=db_model, + updated_patch=updateDeployment(model_name="my-openai/gpt-5.6", model_info=ModelInfo(**echo)), + llm_router=router, + ) + + info: Final = json.loads(result["model_info"]) + assert info["access_groups"] == ["prod"] + assert set(info).isdisjoint(entry), "one listed model's metadata must not be saved onto the whole wildcard" + + def test_echo_saved_on_a_worker_that_has_not_loaded_the_wildcard_is_not_persisted(self): + import litellm + from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model + from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo + + router_without_the_wildcard: Final = litellm.Router( + model_list=[ + { + "model_name": "openai-mini", + "litellm_params": {"model": "openai/gpt-5.4-mini", "api_key": "x"}, + "model_info": {"id": "dep-other"}, + } + ] + ) + entry: Final = litellm.get_model_info("openai/gpt-5.6") + db_model: Final = Deployment( + model_name="my-openai/*", + litellm_params=LiteLLM_Params(model="openai/*"), + model_info=ModelInfo(id="dep-echo-wildcard"), + ) + + result: Final = update_db_model( + db_model=db_model, + updated_patch=updateDeployment( + model_name="my-openai/gpt-5.6", + model_info=ModelInfo(**{**entry, "id": "dep-echo-wildcard", "access_groups": ["prod"]}), + ), + llm_router=router_without_the_wildcard, + ) + + info: Final = json.loads(result["model_info"]) + assert info["access_groups"] == ["prod"] + assert set(info).isdisjoint(entry), "one listed model's metadata must not be saved onto the whole wildcard" + + def test_legacy_update_echo_of_an_expanded_wildcard_row_is_not_persisted_on_the_wildcard(self): + import litellm + from litellm.proxy.management_endpoints.model_management_endpoints import ( + _ListedRow, + legacy_update_model_info_changes, + ) + from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo + + router: Final = litellm.Router( + model_list=[ + { + "model_name": "my-openai/*", + "litellm_params": {"model": "openai/*", "api_key": "x"}, + "model_info": {"id": "dep-echo-wildcard"}, + } + ] + ) + entry: Final = litellm.get_model_info("openai/gpt-5.6") + db_model: Final = Deployment( + model_name="my-openai/*", + litellm_params=LiteLLM_Params(model="openai/*"), + model_info=ModelInfo(id="dep-echo-wildcard"), + ) + + changes: Final = legacy_update_model_info_changes( + db_model, + ModelInfo(**{**entry, "id": "dep-echo-wildcard", "access_groups": ["prod"]}), + loaded_catalog=lambda: {}, + listed_row=_ListedRow("my-openai/gpt-5.6", router), + ) + + assert dict(changes) == {"access_groups": ["prod"]} + + @pytest.mark.asyncio + async def test_echo_of_a_team_wildcard_row_is_not_persisted_on_the_wildcard(self): + import litellm + from litellm.proxy.management_endpoints.model_management_endpoints import _update_team_model_in_db + from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo + + router: Final = litellm.Router( + model_list=[ + { + "model_name": "team-openai/*", + "litellm_params": {"model": "openai/*", "api_key": "x"}, + "model_info": {"id": "dep-team-wildcard", "team_id": "team-a"}, + } + ] + ) + entry: Final = litellm.get_model_info("openai/gpt-5.6") + echo: Final = {**entry, "id": "dep-team-wildcard", "team_id": "team-a", "access_groups": ["prod"]} + db_model: Final = Deployment( + model_name="team-openai/*", + litellm_params=LiteLLM_Params(model="openai/*"), + model_info=ModelInfo(id="dep-team-wildcard", team_id="team-a"), + ) + + with ( + patch("litellm.proxy.proxy_server.premium_user", True), + patch("litellm.proxy.management_endpoints.model_management_endpoints.team_model_add"), + patch("litellm.proxy.management_endpoints.model_management_endpoints.team_model_delete"), + ): + result: Final = await _update_team_model_in_db( + db_model=db_model, + patch_data=updateDeployment(model_name="team-openai/gpt-5.6", model_info=ModelInfo(**echo)), + user_api_key_dict=UserAPIKeyAuth(user_id="test_user", user_role=LitellmUserRoles.PROXY_ADMIN), + prisma_client=MockPrismaClient(team_exists=True), + write_row=_passthrough_row, + llm_router=router, + ) + + info: Final = json.loads(result["model_info"]) + assert info["access_groups"] == ["prod"] + assert set(info).isdisjoint(entry), "one listed model's metadata must not be saved onto a team's wildcard" + def test_an_edited_value_survives_the_echo_filter(self): import litellm from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model diff --git a/tests/unit/proxy/proxy_server/test_routes_model_info.py b/tests/unit/proxy/proxy_server/test_routes_model_info.py index 79f3e697a4f..f3fe1e2520a 100644 --- a/tests/unit/proxy/proxy_server/test_routes_model_info.py +++ b/tests/unit/proxy/proxy_server/test_routes_model_info.py @@ -12,7 +12,7 @@ from __future__ import annotations import asyncio import copy import json -from collections.abc import Callable +from collections.abc import Callable, Mapping from contextlib import AbstractContextManager from typing import Final from unittest.mock import AsyncMock, MagicMock @@ -26,7 +26,7 @@ import litellm from litellm.caching.llm_caching_handler import LLMClientCache from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.proxy import proxy_server -from litellm.proxy._types import LiteLLM_ProxyModelTable +from litellm.proxy._types import LiteLLM_ProxyModelTable, UserAPIKeyAuth from litellm.utils import _invalidate_model_cost_lowercase_map from .conftest import normalize, route_entry_points_to_stubs # type: ignore[import-not-found] @@ -315,7 +315,7 @@ def configured_router(monkeypatch): monkeypatch.setattr(proxy_server, "llm_router", router) monkeypatch.setattr(proxy_server, "llm_model_list", [{"model_name": "gpt-4"}]) monkeypatch.setattr(proxy_server, "user_model", None) - monkeypatch.setattr(proxy_server, "_get_proxy_model_info", lambda model: model) + monkeypatch.setattr(proxy_server, "_get_proxy_model_info", lambda model, caller_aliases: model) yield router @@ -585,6 +585,11 @@ def test_model_info_id_lookup_reports_the_same_cost_as_the_list( _invalidate_model_cost_lowercase_map() +def _discoverable_models(monkeypatch: pytest.MonkeyPatch, models: Mapping[str, list[str]]) -> None: + for provider, provider_models in models.items(): + monkeypatch.setitem(litellm.models_by_provider, provider, provider_models) + + async def test_v1_model_info_star_wildcard_filter_keeps_provider_expansion(monkeypatch): from litellm.proxy._types import SpecialModelNames, UserAPIKeyAuth from litellm.proxy.auth import model_checks @@ -671,6 +676,839 @@ def test_v1_model_info_asks_a_wildcard_server_only_for_keys_that_can_call_it( ) == (expected_names, expected_listings, 0) +def test_model_info_prices_expanded_wildcard_rows_as_the_model_their_calls_route_to( + client: TestClient, + auth_as: Callable[[], AbstractContextManager[object]], + monkeypatch: pytest.MonkeyPatch, + local_model_cost_map: None, +) -> None: + """A catch-all over Fireworks lists ``acme-chat``, which also names a DeepSeek entry, and a + prefixed OpenAI wildcard lists ``my-openai/gpt-acme``, which names no entry at all. Each row must + show the provider and price its calls are billed at.""" + + catalog: Final = { + "fireworks_ai/accounts/fireworks/models/acme-chat": { + "input_cost_per_token": 1e-07, + "output_cost_per_token": 2e-07, + "litellm_provider": "fireworks_ai", + "mode": "chat", + }, + "acme-chat": { + "input_cost_per_token": 9e-07, + "output_cost_per_token": 9e-06, + "litellm_provider": "deepseek", + "mode": "chat", + }, + "gpt-acme": { + "input_cost_per_token": 3e-07, + "output_cost_per_token": 4e-07, + "litellm_provider": "openai", + "mode": "chat", + }, + } + monkeypatch.setattr(litellm, "model_cost", {**litellm.model_cost, **catalog}) + _invalidate_model_cost_lowercase_map() + router: Final = litellm.Router( + model_list=[ + { + "model_name": "*", + "litellm_params": {"model": "fireworks_ai/accounts/fireworks/models/*", "api_key": "x"}, + "model_info": {"id": "fireworks-wildcard"}, + }, + { + "model_name": "my-openai/*", + "litellm_params": {"model": "openai/*", "api_key": "x"}, + "model_info": {"id": "openai-wildcard"}, + }, + ] + ) + monkeypatch.setattr(proxy_server, "llm_router", router) + _discoverable_models( + monkeypatch, + { + "fireworks_ai": ["fireworks_ai/accounts/fireworks/models/acme-chat"], + "openai": ["gpt-acme"], + }, + ) + monkeypatch.setattr(proxy_server, "llm_model_list", router.get_model_list()) + monkeypatch.setattr(proxy_server, "user_model", None) + + with auth_as(): + response: Final = client.get("/model/info") + + assert response.status_code == 200, response.text + listed: Final = { + row["model_name"]: ( + row["litellm_params"]["model"], + row["model_info"].get("litellm_provider"), + row["model_info"].get("input_cost_per_token"), + row["model_info"].get("output_cost_per_token"), + ) + for row in response.json()["data"] + } + assert listed == { + "acme-chat": ("fireworks_ai/accounts/fireworks/models/acme-chat", "fireworks_ai", 1e-07, 2e-07), + "my-openai/gpt-acme": ("openai/gpt-acme", "openai", 3e-07, 4e-07), + } + calls: Final = { + model_name: router.completion( + model=model_name, messages=[{"role": "user", "content": "hi"}], mock_response="hello there" + ) + for model_name in listed + } + assert {model_name: call._hidden_params["response_cost"] for model_name, call in calls.items()} == pytest.approx( + { + model_name: listed[model_name][2] * call.usage.prompt_tokens + + listed[model_name][3] * call.usage.completion_tokens + for model_name, call in calls.items() + } + ) + _invalidate_model_cost_lowercase_map() + + +def test_model_info_prices_a_wildcard_row_as_the_other_deployment_its_calls_reach( + client: TestClient, + auth_as: Callable[[], AbstractContextManager[object]], + monkeypatch: pytest.MonkeyPatch, + local_model_cost_map: None, +) -> None: + """``acme-pro`` is an alias of the ``acme-lite`` deployment and ``acme-max`` is a routing group over it, so + calls to those names never reach the catch-all, which still lists them. Such a row shows the provider and + price of the deployment its calls reach, even where another provider's model goes by the listed name.""" + + catalog: Final = { + "fireworks_ai/accounts/fireworks/models/acme-lite": { + "input_cost_per_token": 5e-07, + "output_cost_per_token": 6e-07, + "litellm_provider": "fireworks_ai", + "mode": "chat", + }, + "fireworks_ai/accounts/fireworks/models/acme-pro": { + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 2.5e-07, + "litellm_provider": "fireworks_ai", + "mode": "chat", + }, + "fireworks_ai/accounts/fireworks/models/acme-max": { + "input_cost_per_token": 3.5e-07, + "output_cost_per_token": 4.5e-07, + "litellm_provider": "fireworks_ai", + "mode": "chat", + }, + "deepseek/acme-lite": { + "input_cost_per_token": 7e-07, + "output_cost_per_token": 8e-07, + "litellm_provider": "deepseek", + "mode": "chat", + }, + "acme-pro": { + "input_cost_per_token": 9e-07, + "output_cost_per_token": 1e-06, + "litellm_provider": "deepseek", + "mode": "chat", + }, + "acme-max": { + "input_cost_per_token": 1.1e-06, + "output_cost_per_token": 1.2e-06, + "litellm_provider": "deepseek", + "mode": "chat", + }, + } + monkeypatch.setattr(litellm, "model_cost", {**litellm.model_cost, **catalog}) + _invalidate_model_cost_lowercase_map() + router: Final = litellm.Router( + model_list=[ + { + "model_name": "*", + "litellm_params": {"model": "fireworks_ai/accounts/fireworks/models/*", "api_key": "x"}, + "model_info": {"id": "fireworks-wildcard"}, + }, + { + "model_name": "acme-lite", + "litellm_params": {"model": "deepseek/acme-lite", "api_key": "x"}, + "model_info": {"id": "deepseek-acme-lite"}, + }, + ], + model_group_alias={"acme-pro": "acme-lite"}, + routing_groups=[{"group_name": "acme-max", "models": ["acme-lite"], "routing_strategy": "simple-shuffle"}], + ) + monkeypatch.setattr(proxy_server, "llm_router", router) + _discoverable_models( + monkeypatch, + { + "fireworks_ai": [ + "fireworks_ai/accounts/fireworks/models/acme-lite", + "fireworks_ai/accounts/fireworks/models/acme-pro", + "fireworks_ai/accounts/fireworks/models/acme-max", + ], + }, + ) + monkeypatch.setattr(proxy_server, "llm_model_list", router.get_model_list()) + monkeypatch.setattr(proxy_server, "user_model", None) + + with auth_as(): + response: Final = client.get("/model/info") + + assert response.status_code == 200, response.text + listed: Final = { + (row["model_info"]["id"], row["model_name"]): ( + row["model_info"].get("input_cost_per_token"), + row["model_info"].get("output_cost_per_token"), + ) + for row in response.json()["data"] + if "*" not in row["model_name"] + } + assert sorted(listed) == [ + ("deepseek-acme-lite", "acme-lite"), + ("deepseek-acme-lite", "acme-pro"), + ("fireworks-wildcard", "acme-max"), + ("fireworks-wildcard", "acme-pro"), + ] + shown_models: Final = { + (row["model_info"]["id"], row["model_name"]): ( + row["model_info"].get("litellm_provider"), + row["model_info"].get("key"), + ) + for row in response.json()["data"] + if "*" not in row["model_name"] and row["model_info"]["id"] != "deepseek-acme-lite" + } + assert {provider for provider, _ in shown_models.values()} == {"deepseek"} + assert listed[("deepseek-acme-lite", "acme-lite")] == (7e-07, 8e-07) + assert {key: prices for key, prices in listed.items() if key[0] != "deepseek-acme-lite"} == { + ("fireworks-wildcard", "acme-max"): (7e-07, 8e-07), + ("fireworks-wildcard", "acme-pro"): (7e-07, 8e-07), + } + calls: Final = { + model_name: router.completion( + model=model_name, messages=[{"role": "user", "content": "hi"}], mock_response="hello there" + ) + for _, model_name in listed + } + assert {key: calls[key[1]]._hidden_params["response_cost"] for key in listed} == pytest.approx( + { + key: input_cost * calls[key[1]].usage.prompt_tokens + output_cost * calls[key[1]].usage.completion_tokens + for key, (input_cost, output_cost) in listed.items() + } + ) + _invalidate_model_cost_lowercase_map() + + +def _catch_all_and_shadowed_wildcard_router(monkeypatch: pytest.MonkeyPatch) -> litellm.Router: + + catalog: Final = { + "gpt-5.6-acme": { + "input_cost_per_token": 3e-07, + "output_cost_per_token": 4e-07, + "litellm_provider": "openai", + "mode": "chat", + }, + "fireworks_ai/accounts/fireworks/models/acme-pro": { + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 2.5e-07, + "litellm_provider": "fireworks_ai", + "mode": "chat", + }, + "fireworks_ai/accounts/fireworks/models/acme-max": { + "input_cost_per_token": 9e-07, + "output_cost_per_token": 9.5e-07, + "litellm_provider": "fireworks_ai", + "mode": "chat", + }, + "fireworks_ai/accounts/fireworks/models/acme-mini": { + "input_cost_per_token": 5e-08, + "output_cost_per_token": 6e-08, + "litellm_provider": "fireworks_ai", + "mode": "chat", + }, + "deepseek/acme-lite": { + "input_cost_per_token": 7e-07, + "output_cost_per_token": 8e-07, + "litellm_provider": "deepseek", + "mode": "chat", + }, + } + monkeypatch.setattr(litellm, "model_cost", {**litellm.model_cost, **catalog}) + _invalidate_model_cost_lowercase_map() + router: Final = litellm.Router( + model_list=[ + { + "model_name": "*", + "litellm_params": {"model": "openai/*", "api_key": "x"}, + "model_info": {"id": "openai-catch-all"}, + }, + { + "model_name": "my-fw/*", + "litellm_params": {"model": "fireworks_ai/accounts/fireworks/models/*", "api_key": "x"}, + "model_info": {"id": "fireworks-wildcard"}, + }, + { + "model_name": "acme-lite", + "litellm_params": {"model": "deepseek/acme-lite", "api_key": "x"}, + "model_info": {"id": "deepseek-acme-lite"}, + }, + ], + model_group_alias={"my-fw/acme-pro": "acme-lite", "my-fw/acme-max": "my-fw/acme-mini"}, + ) + monkeypatch.setattr(proxy_server, "llm_router", router) + _discoverable_models( + monkeypatch, + { + "openai": ["gpt-5.6-acme"], + "fireworks_ai": [ + "fireworks_ai/accounts/fireworks/models/acme-pro", + "fireworks_ai/accounts/fireworks/models/acme-max", + "fireworks_ai/accounts/fireworks/models/acme-mini", + ], + }, + ) + monkeypatch.setattr(proxy_server, "llm_model_list", router.get_model_list()) + monkeypatch.setattr(proxy_server, "user_model", None) + return router + + +def _expanded_wildcard_rows(client: TestClient) -> dict[str, dict[str, object]]: + response: Final = client.get("/model/info") + assert response.status_code == 200, response.text + return { + row["model_name"]: row + for row in response.json()["data"] + if row["model_info"]["id"] != "deepseek-acme-lite" and "*" not in row["model_name"] + } + + +def test_model_info_prices_a_wildcard_row_aliased_to_a_sibling_name_as_the_alias_target( + client: TestClient, + auth_as: Callable[[], AbstractContextManager[object]], + monkeypatch: pytest.MonkeyPatch, + local_model_cost_map: None, +) -> None: + """An alias from ``my-fw/acme-max`` to ``my-fw/acme-mini`` still reaches the same wildcard, but calls run + ``acme-mini``, so the ``my-fw/acme-max`` row must show the price those calls are billed at.""" + router: Final = _catch_all_and_shadowed_wildcard_router(monkeypatch) + + with auth_as(): + rows: Final = _expanded_wildcard_rows(client) + + shown: Final = rows["my-fw/acme-max"]["model_info"] + call: Final = router.completion( + model="my-fw/acme-max", messages=[{"role": "user", "content": "hi"}], mock_response="hello there" + ) + assert call._hidden_params["response_cost"] == pytest.approx( + shown["input_cost_per_token"] * call.usage.prompt_tokens + + shown["output_cost_per_token"] * call.usage.completion_tokens + ) + _invalidate_model_cost_lowercase_map() + + +def test_model_info_prices_a_wildcard_row_the_proxy_renames_as_the_model_its_calls_reach( + client: TestClient, + auth_as: Callable[..., AbstractContextManager[object]], + monkeypatch: pytest.MonkeyPatch, + local_model_cost_map: None, +) -> None: + """The proxy renames a requested model through the global ``model_alias_map``, the key's ``aliases`` and the + key's ``router_settings.model_group_alias`` before the router sees it. A row renamed to another deployment + shows that deployment's price and provider, and a row renamed to a sibling name of its own wildcard shows + that sibling's price.""" + _catch_all_and_shadowed_wildcard_router(monkeypatch) + monkeypatch.setattr(litellm, "model_alias_map", {"my-fw/acme-max": "acme-lite"}) + + with auth_as( + aliases={"gpt-5.6-acme": "acme-lite"}, + router_settings={"model_group_alias": {"my-fw/acme-pro": "my-fw/acme-mini"}}, + ): + rows: Final = _expanded_wildcard_rows(client) + + shown: Final = { + name: ( + row["model_info"].get("input_cost_per_token"), + row["model_info"].get("output_cost_per_token"), + row["model_info"].get("litellm_provider"), + ) + for name, row in rows.items() + } + assert shown == { + "my-fw/acme-max": (7e-07, 8e-07, "deepseek"), + "my-fw/acme-mini": (5e-08, 6e-08, "fireworks_ai"), + "my-fw/acme-pro": (5e-08, 6e-08, "fireworks_ai"), + "gpt-5.6-acme": (7e-07, 8e-07, "deepseek"), + } + _invalidate_model_cost_lowercase_map() + + +def test_model_info_prices_a_wildcard_row_whose_team_alias_targets_a_deleted_team_deployment( + client: TestClient, + auth_as: Callable[..., AbstractContextManager[object]], + monkeypatch: pytest.MonkeyPatch, + local_model_cost_map: None, +) -> None: + """Calls skip a team alias whose target team deployment was deleted, so the row that alias names keeps the + price its calls through the wildcard are billed at, while the team's live aliases still apply.""" + from litellm.proxy.litellm_pre_call_utils import _update_model_if_team_alias_exists + + _catch_all_and_shadowed_wildcard_router(monkeypatch) + caller: Final = { + "team_id": "team-a", + "team_model_aliases": {"my-fw/acme-mini": "model_name_team-a_deleted", "gpt-5.6-acme": "my-fw/acme-mini"}, + } + + with auth_as(**caller): + rows: Final = _expanded_wildcard_rows(client) + + stale_request: Final = {"model": "my-fw/acme-mini"} + live_request: Final = {"model": "gpt-5.6-acme"} + _update_model_if_team_alias_exists(data=stale_request, user_api_key_dict=UserAPIKeyAuth(**caller)) + _update_model_if_team_alias_exists(data=live_request, user_api_key_dict=UserAPIKeyAuth(**caller)) + shown: Final = rows["my-fw/acme-mini"]["model_info"] + live_alias_row: Final = rows["gpt-5.6-acme"]["model_info"] + assert (stale_request["model"], live_request["model"]) == ("my-fw/acme-mini", "my-fw/acme-mini") + assert ( + shown.get("input_cost_per_token"), + shown.get("output_cost_per_token"), + shown.get("litellm_provider"), + ) == (5e-08, 6e-08, "fireworks_ai") + assert (live_alias_row.get("input_cost_per_token"), live_alias_row.get("litellm_provider")) == ( + 5e-08, + "fireworks_ai", + ) + _invalidate_model_cost_lowercase_map() + + +def _fireworks_wildcard_rows(client: TestClient) -> dict[str, dict[str, object]]: + response: Final = client.get("/model/info") + assert response.status_code == 200, response.text + return { + row["model_name"]: row["model_info"] + for row in response.json()["data"] + if row["model_info"]["id"] == "fireworks-wildcard" and "*" not in row["model_name"] + } + + +_FIREWORKS_WILDCARD: Final = { + "model_name": "my-fw/*", + "litellm_params": {"model": "fireworks_ai/accounts/fireworks/models/*", "api_key": "x"}, + "model_info": {"id": "fireworks-wildcard"}, +} + + +def test_model_info_prices_a_wildcard_row_as_the_deployment_the_callers_team_publishes_under_its_name( + client: TestClient, + auth_as: Callable[..., AbstractContextManager[object]], + monkeypatch: pytest.MonkeyPatch, + local_model_cost_map: None, +) -> None: + """Team ``team-a`` publishes ``my-fw/acme-mini`` on its own DeepSeek deployment, so a team-a key's calls to that + name skip the global wildcard and the wildcard's row shows that key the DeepSeek price, while a key outside + the team still sees the Fireworks price its calls bill.""" + _catch_all_and_shadowed_wildcard_router(monkeypatch) + router: Final = litellm.Router( + model_list=[ + _FIREWORKS_WILDCARD, + { + "model_name": "model_name_team-a_acme-mini", + "litellm_params": {"model": "deepseek/acme-lite", "api_key": "x"}, + "model_info": {"id": "team-deepseek", "team_id": "team-a", "team_public_model_name": "my-fw/acme-mini"}, + }, + ] + ) + monkeypatch.setattr(proxy_server, "llm_router", router) + _discoverable_models(monkeypatch, {"fireworks_ai": ["fireworks_ai/accounts/fireworks/models/acme-mini"]}) + monkeypatch.setattr(proxy_server, "llm_model_list", router.get_model_list()) + + with auth_as(team_id="team-a"): + team_rows: Final = _fireworks_wildcard_rows(client) + with auth_as(): + other_rows: Final = _fireworks_wildcard_rows(client) + + assert team_rows["my-fw/acme-mini"].get("input_cost_per_token") == 7e-07 + assert other_rows["my-fw/acme-mini"].get("input_cost_per_token") == 5e-08 + _invalidate_model_cost_lowercase_map() + + +def _call_cost_at_row_rates(call: litellm.ModelResponse, row: Mapping[str, float]) -> float: + return ( + row["input_cost_per_token"] * call.usage.prompt_tokens + + row["output_cost_per_token"] * call.usage.completion_tokens + ) + + +@pytest.mark.parametrize( + ("wildcards", "row_ids"), + [ + pytest.param( + (("gpt-*", "openai/gpt-*", "narrow"), ("*", "openai/*", "catch-all")), + {"narrow", "catch-all"}, + id="catch_all_row_a_narrower_wildcard_serves", + ), + pytest.param( + (("oa/*", "openai/*", "wide"), ("oa/gpt-*", "openai/gpt-*", "narrow")), + {"wide", "narrow"}, + id="overlapping_wildcards", + ), + ], +) +def test_model_info_keeps_the_price_and_limits_of_a_row_another_wildcard_serves( + client: TestClient, + auth_as: Callable[[], AbstractContextManager[object]], + monkeypatch: pytest.MonkeyPatch, + local_model_cost_map: None, + wildcards: tuple[tuple[str, str, str], ...], + row_ids: set[str], +) -> None: + """Two wildcards can list one name while calls reach only one of them. Each row keeps the price and limits of + the model those calls bill, so a client that reads the first row by name still gets them.""" + entry: Final = { + "input_cost_per_token": 3e-07, + "output_cost_per_token": 4e-07, + "max_input_tokens": 4321, + "litellm_provider": "openai", + "mode": "chat", + } + monkeypatch.setattr(litellm, "model_cost", {**litellm.model_cost, "gpt-5.6-acme": entry}) + _invalidate_model_cost_lowercase_map() + router: Final = litellm.Router( + model_list=[ + { + "model_name": model_name, + "litellm_params": {"model": litellm_model, "api_key": "x"}, + "model_info": {"id": model_id}, + } + for model_name, litellm_model, model_id in wildcards + ] + ) + monkeypatch.setattr(proxy_server, "llm_router", router) + _discoverable_models(monkeypatch, {"openai": ["gpt-5.6-acme"]}) + monkeypatch.setattr(proxy_server, "llm_model_list", router.get_model_list()) + monkeypatch.setattr(proxy_server, "user_model", None) + + with auth_as(): + response: Final = client.get("/model/info") + + assert response.status_code == 200, response.text + rows: Final = [row for row in response.json()["data"] if "*" not in row["model_name"]] + assert {row["model_info"]["id"] for row in rows} == row_ids + assert {(row["model_info"]["input_cost_per_token"], row["model_info"]["max_input_tokens"]) for row in rows} == { + (3e-07, 4321) + } + call: Final = router.completion( + model=rows[0]["model_name"], messages=[{"role": "user", "content": "hi"}], mock_response="hello there" + ) + assert call._hidden_params["response_cost"] == pytest.approx(_call_cost_at_row_rates(call, rows[0]["model_info"])) + _invalidate_model_cost_lowercase_map() + + +def test_model_info_prices_an_aliased_wildcard_row_as_the_model_the_other_wildcard_bills( + client: TestClient, + auth_as: Callable[[], AbstractContextManager[object]], + monkeypatch: pytest.MonkeyPatch, + local_model_cost_map: None, +) -> None: + """``based/gpt-5.6-acme`` is aliased to a name another wildcard serves, so its row shows the price of the model + the other wildcard fills in from the name, which is what its calls bill.""" + catalog: Final = { + "gpt-5.6-acme": { + "input_cost_per_token": 3e-07, + "output_cost_per_token": 4e-07, + "litellm_provider": "openai", + "mode": "chat", + }, + "gpt-5.6-acme-mini": { + "input_cost_per_token": 3e-08, + "output_cost_per_token": 4e-08, + "litellm_provider": "openai", + "mode": "chat", + }, + } + monkeypatch.setattr(litellm, "model_cost", {**litellm.model_cost, **catalog}) + _invalidate_model_cost_lowercase_map() + router: Final = litellm.Router( + model_list=[ + { + "model_name": name_pattern, + "litellm_params": {"model": "openai/*", "api_key": "x"}, + "model_info": {"id": name_pattern}, + } + for name_pattern in ("based/*", "other/*") + ], + model_group_alias={"based/gpt-5.6-acme": "other/gpt-5.6-acme-mini"}, + ) + monkeypatch.setattr(proxy_server, "llm_router", router) + _discoverable_models(monkeypatch, {"openai": ["gpt-5.6-acme"]}) + monkeypatch.setattr(proxy_server, "llm_model_list", router.get_model_list()) + monkeypatch.setattr(proxy_server, "user_model", None) + + with auth_as(): + response: Final = client.get("/model/info") + + assert response.status_code == 200, response.text + aliased: Final = next(row for row in response.json()["data"] if row["model_name"] == "based/gpt-5.6-acme") + call: Final = router.completion( + model="based/gpt-5.6-acme", messages=[{"role": "user", "content": "hi"}], mock_response="hello there" + ) + assert aliased["model_info"]["input_cost_per_token"] == 3e-08 + assert call._hidden_params["response_cost"] == pytest.approx(_call_cost_at_row_rates(call, aliased["model_info"])) + _invalidate_model_cost_lowercase_map() + + +def test_model_info_keeps_the_price_of_a_wildcard_row_whose_listed_name_no_route_serves( + client: TestClient, + auth_as: Callable[[], AbstractContextManager[object]], + monkeypatch: pytest.MonkeyPatch, + local_model_cost_map: None, +) -> None: + """``my-*`` expands to provider-prefixed names no route matches, so nothing says the row's price is wrong and + the row keeps the price of the model it lists.""" + entry: Final = { + "input_cost_per_token": 3e-07, + "output_cost_per_token": 4e-07, + "litellm_provider": "openai", + "mode": "chat", + } + monkeypatch.setattr(litellm, "model_cost", {**litellm.model_cost, "gpt-5.6-acme": entry}) + _invalidate_model_cost_lowercase_map() + router: Final = litellm.Router( + model_list=[ + { + "model_name": "my-*", + "litellm_params": {"model": "openai/*", "api_key": "x"}, + "model_info": {"id": "my-wildcard"}, + } + ] + ) + monkeypatch.setattr(proxy_server, "llm_router", router) + _discoverable_models(monkeypatch, {"openai": ["gpt-5.6-acme"]}) + monkeypatch.setattr(proxy_server, "llm_model_list", router.get_model_list()) + monkeypatch.setattr(proxy_server, "user_model", None) + + with auth_as(): + response: Final = client.get("/model/info") + + assert response.status_code == 200, response.text + assert { + row["model_info"].get("input_cost_per_token") for row in response.json()["data"] if "*" not in row["model_name"] + } == {3e-07} + _invalidate_model_cost_lowercase_map() + + +@pytest.mark.parametrize( + "targets", + [ + pytest.param( + ( + { + "litellm_params": {"model": "deepseek/acme-lite", "api_key": "x", "input_cost_per_token": 9e-06}, + "model_info": {"id": "target"}, + }, + ), + id="target_with_its_own_rate", + ), + pytest.param( + ( + {"litellm_params": {"model": "deepseek/acme-lite", "api_key": "x"}, "model_info": {"id": "target-a"}}, + {"litellm_params": {"model": "deepseek/acme-big", "api_key": "x"}, "model_info": {"id": "target-b"}}, + ), + id="targets_that_bill_different_models", + ), + ], +) +def test_model_info_shows_no_wildcard_price_when_the_cost_map_cannot_say_what_calls_bill( + client: TestClient, + auth_as: Callable[[], AbstractContextManager[object]], + monkeypatch: pytest.MonkeyPatch, + local_model_cost_map: None, + targets: tuple[Mapping[str, object], ...], +) -> None: + """An alias sends ``my-fw/acme-mini`` to a group whose bill the cost map cannot state as one price, so the + wildcard's row shows no price rather than one calls are not billed at.""" + catalog: Final = { + "deepseek/acme-lite": { + "input_cost_per_token": 7e-07, + "output_cost_per_token": 8e-07, + "litellm_provider": "deepseek", + "mode": "chat", + }, + "deepseek/acme-big": { + "input_cost_per_token": 1.7e-06, + "output_cost_per_token": 1.8e-06, + "litellm_provider": "deepseek", + "mode": "chat", + }, + } + monkeypatch.setattr(litellm, "model_cost", {**litellm.model_cost, **catalog}) + _invalidate_model_cost_lowercase_map() + router: Final = litellm.Router( + model_list=[_FIREWORKS_WILDCARD, *({"model_name": "acme-target", **target} for target in targets)], + model_group_alias={"my-fw/acme-mini": "acme-target"}, + ) + monkeypatch.setattr(proxy_server, "llm_router", router) + _discoverable_models(monkeypatch, {"fireworks_ai": ["fireworks_ai/accounts/fireworks/models/acme-mini"]}) + monkeypatch.setattr(proxy_server, "llm_model_list", router.get_model_list()) + monkeypatch.setattr(proxy_server, "user_model", None) + + with auth_as(): + rows: Final = _fireworks_wildcard_rows(client) + + assert rows["my-fw/acme-mini"].get("input_cost_per_token") is None + _invalidate_model_cost_lowercase_map() + + +def test_model_info_prices_a_wildcard_row_aliased_to_an_aliased_group_as_the_group_its_calls_reach( + client: TestClient, + auth_as: Callable[[], AbstractContextManager[object]], + monkeypatch: pytest.MonkeyPatch, + local_model_cost_map: None, +) -> None: + """``my-fw/acme-mini`` is aliased to ``acme-target``, a group that is itself aliased onward. Calls follow one + alias and reach ``acme-target``'s deployment, so the row shows that deployment's price, not the onward one's.""" + catalog: Final = { + "deepseek/acme-lite": { + "input_cost_per_token": 7e-07, + "output_cost_per_token": 8e-07, + "litellm_provider": "deepseek", + "mode": "chat", + }, + "deepseek/acme-big": { + "input_cost_per_token": 1.7e-06, + "output_cost_per_token": 1.8e-06, + "litellm_provider": "deepseek", + "mode": "chat", + }, + } + monkeypatch.setattr(litellm, "model_cost", {**litellm.model_cost, **catalog}) + _invalidate_model_cost_lowercase_map() + router: Final = litellm.Router( + model_list=[ + _FIREWORKS_WILDCARD, + { + "model_name": "acme-target", + "litellm_params": {"model": "deepseek/acme-lite", "api_key": "x"}, + "model_info": {"id": "target"}, + }, + { + "model_name": "acme-onward", + "litellm_params": {"model": "deepseek/acme-big", "api_key": "x"}, + "model_info": {"id": "onward"}, + }, + ], + model_group_alias={"my-fw/acme-mini": "acme-target", "acme-target": "acme-onward"}, + ) + monkeypatch.setattr(proxy_server, "llm_router", router) + _discoverable_models(monkeypatch, {"fireworks_ai": ["fireworks_ai/accounts/fireworks/models/acme-mini"]}) + monkeypatch.setattr(proxy_server, "llm_model_list", router.get_model_list()) + monkeypatch.setattr(proxy_server, "user_model", None) + + with auth_as(): + rows: Final = _fireworks_wildcard_rows(client) + + call: Final = router.completion( + model="my-fw/acme-mini", messages=[{"role": "user", "content": "hi"}], mock_response="hello there" + ) + assert rows["my-fw/acme-mini"].get("input_cost_per_token") is not None + assert call._hidden_params["response_cost"] == pytest.approx(_call_cost_at_row_rates(call, rows["my-fw/acme-mini"])) + _invalidate_model_cost_lowercase_map() + + +def test_model_info_prices_a_wildcard_row_aliased_to_another_deployment_as_that_deployment_not_the_wildcards_base_model( + client: TestClient, + auth_as: Callable[[], AbstractContextManager[object]], + monkeypatch: pytest.MonkeyPatch, + local_model_cost_map: None, +) -> None: + """The wildcard declares a ``base_model`` for its own calls, but ``my-fw/acme-mini`` is aliased to another + deployment, so the row shows the price that deployment bills.""" + catalog: Final = { + "deepseek/acme-lite": { + "input_cost_per_token": 7e-07, + "output_cost_per_token": 8e-07, + "litellm_provider": "deepseek", + "mode": "chat", + }, + "deepseek/acme-base": { + "input_cost_per_token": 9e-06, + "output_cost_per_token": 9.5e-06, + "litellm_provider": "deepseek", + "mode": "chat", + }, + } + monkeypatch.setattr(litellm, "model_cost", {**litellm.model_cost, **catalog}) + _invalidate_model_cost_lowercase_map() + router: Final = litellm.Router( + model_list=[ + {**_FIREWORKS_WILDCARD, "model_info": {"id": "fireworks-wildcard", "base_model": "deepseek/acme-base"}}, + { + "model_name": "acme-target", + "litellm_params": {"model": "deepseek/acme-lite", "api_key": "x"}, + "model_info": {"id": "target"}, + }, + ], + model_group_alias={"my-fw/acme-mini": "acme-target"}, + ) + monkeypatch.setattr(proxy_server, "llm_router", router) + _discoverable_models(monkeypatch, {"fireworks_ai": ["fireworks_ai/accounts/fireworks/models/acme-mini"]}) + monkeypatch.setattr(proxy_server, "llm_model_list", router.get_model_list()) + monkeypatch.setattr(proxy_server, "user_model", None) + + with auth_as(): + rows: Final = _fireworks_wildcard_rows(client) + + call: Final = router.completion( + model="my-fw/acme-mini", messages=[{"role": "user", "content": "hi"}], mock_response="hello there" + ) + assert rows["my-fw/acme-mini"].get("input_cost_per_token") is not None + assert call._hidden_params["response_cost"] == pytest.approx(_call_cost_at_row_rates(call, rows["my-fw/acme-mini"])) + _invalidate_model_cost_lowercase_map() + + +def test_saving_a_replayed_model_info_wildcard_row_stores_none_of_its_shown_metadata( + client: TestClient, + auth_as: Callable[[], AbstractContextManager[object]], + monkeypatch: pytest.MonkeyPatch, + local_model_cost_map: None, +) -> None: + """An API client that saves a ``/model/info`` row back unchanged, apart from one edit, must not pin the + cost-map metadata the row showed onto the wildcard, whether the row is a catch-all name or a name an alias + sends to a sibling name or to another deployment.""" + from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model + from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo, updateDeployment + + router: Final = _catch_all_and_shadowed_wildcard_router(monkeypatch) + with auth_as(): + rows: Final = _expanded_wildcard_rows(client) + + assert sorted(rows) == ["gpt-5.6-acme", "my-fw/acme-max", "my-fw/acme-mini", "my-fw/acme-pro"] + sources: Final = { + deployment["model_info"]["id"]: deployment for deployment in router.get_model_list() or [] + } + stored: Final = { + name: json.loads( + update_db_model( + db_model=Deployment( + model_name=sources[row["model_info"]["id"]]["model_name"], + litellm_params=LiteLLM_Params(model=sources[row["model_info"]["id"]]["litellm_params"]["model"]), + model_info=ModelInfo(id=row["model_info"]["id"]), + ), + updated_patch=updateDeployment( + model_name=name, model_info=ModelInfo(**{**row["model_info"], "access_groups": ["prod"]}) + ), + llm_router=router, + )["model_info"] + ) + for name, row in rows.items() + } + shown_metadata: Final = { + name: set(row["model_info"]) - set(sources[row["model_info"]["id"]]["model_info"]) for name, row in rows.items() + } + assert sorted(name for name, row in rows.items() if row["model_info"].get("key")) == [ + "gpt-5.6-acme", + "my-fw/acme-max", + "my-fw/acme-mini", + "my-fw/acme-pro", + ] + assert {name: info["access_groups"] for name, info in stored.items()} == dict.fromkeys(rows, ["prod"]) + assert {name: shown_metadata[name] & set(info) for name, info in stored.items()} == dict.fromkeys(rows, set()) + _invalidate_model_cost_lowercase_map() + + # --------------------------------------------------------------------------- # GET /model/info — team BYOK scoping (issue #30983) # --------------------------------------------------------------------------- diff --git a/tests/unit/proxy/proxy_server/test_team_model_name_translation.py b/tests/unit/proxy/proxy_server/test_team_model_name_translation.py index b647fbdd02a..e2b4acc0c09 100644 --- a/tests/unit/proxy/proxy_server/test_team_model_name_translation.py +++ b/tests/unit/proxy/proxy_server/test_team_model_name_translation.py @@ -808,7 +808,7 @@ async def test_model_info_v1_litellm_model_id_include_team_models_filters_inacce monkeypatch.setattr(ps, "llm_model_list", [team_row]) monkeypatch.setattr(ps, "llm_router", router) monkeypatch.setattr(ps, "prisma_client", MagicMock()) - monkeypatch.setattr(ps, "_get_proxy_model_info", lambda model: team_row) + monkeypatch.setattr(ps, "_get_proxy_model_info", lambda model, caller_aliases: team_row) monkeypatch.setattr(ps, "_populate_team_access_on_models", _fake_populate) caller = UserAPIKeyAuth( @@ -843,7 +843,7 @@ async def test_model_info_v1_litellm_model_id_team_id_applies_team_filter(monkey monkeypatch.setattr(ps, "llm_model_list", [team_row]) monkeypatch.setattr(ps, "llm_router", router) monkeypatch.setattr(ps, "prisma_client", MagicMock()) - monkeypatch.setattr(ps, "_get_proxy_model_info", lambda model: team_row) + monkeypatch.setattr(ps, "_get_proxy_model_info", lambda model, caller_aliases: team_row) monkeypatch.setattr(ps, "_populate_team_access_on_models", _fake_populate) monkeypatch.setattr(ps, "_filter_models_by_team_id", team_filter) diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index ca61fcd0905..24ade9b48fb 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -20331,3 +20331,22 @@ def test_router_warns_when_a_deployment_enforces_endpoints_without_a_list(caplog assert "Deployment 'embed-from-config' sets enforce_supported_endpoints without" in caplog.text assert "Deployment 'embed-from-api' sets enforce_supported_endpoints without" in caplog.text + + +def test_registry_finds_a_team_deployment_by_public_and_internal_name_for_its_team_only() -> None: + internal_name: Final = "model_name_team-a_row" + router: Final = Router( + model_list=[ + { + "model_name": internal_name, + "litellm_params": {"model": "openai/gpt-5.5", "api_key": "x"}, + "model_info": {"id": "team-row", "team_id": "team-a", "team_public_model_name": "public-chat"}, + } + ] + ) + registry: Final = router.registry() + + assert [row["model_info"]["id"] for row in registry.deployments("public-chat", team_id="team-a")] == ["team-row"] + assert [row["model_info"]["id"] for row in registry.deployments(internal_name, team_id="team-a")] == ["team-row"] + assert list(registry.deployments("public-chat", team_id="team-b")) == [] + assert list(registry.deployments("public-chat")) == []