diff --git a/litellm/proxy/common_utils/codex_model_catalog.py b/litellm/proxy/common_utils/codex_model_catalog.py index 115b31930dc..825b07b3c3c 100644 --- a/litellm/proxy/common_utils/codex_model_catalog.py +++ b/litellm/proxy/common_utils/codex_model_catalog.py @@ -376,14 +376,14 @@ def codex_models_response_json( def _catalog_row( - row: ModelInfoResponse, lookup_id: str, llm_router: Router | None, team_id: str | None + row: ModelInfoResponse, lookup_id: str, label_id: str, llm_router: Router | None, team_id: str | None ) -> CodexCatalogRow: return CodexCatalogRow( id=row["id"], mode=row.get("mode"), max_input_tokens=row.get("max_input_tokens"), upstream_model=llm_router.get_routable_upstream_model(lookup_id, team_id) if llm_router is not None else None, - display_name=llm_router.get_configured_display_name(lookup_id) if llm_router is not None else None, + display_name=llm_router.get_configured_display_name(label_id) if llm_router is not None else None, service_tiers=llm_router.get_configured_service_tiers(lookup_id, team_id) if llm_router is not None else (), ) @@ -393,14 +393,21 @@ def codex_catalog_rows( entries: Sequence[tuple[str, str]], llm_router: Router | None, team_id: str | None = None, + label_entries: Sequence[tuple[str, str]] | None = None, ) -> tuple[CodexCatalogRow, ...]: """`rows` joined with the router's configured metadata, looked up by each entry's internal id so team-scoped rows resolve the way the Anthropic listing's display names do, a `model_group_alias` reading its target's deployments under its own name; `team_id` is the requesting key's team, so the upstream model that picks Codex's stock entry and the tiers are read only off the deployments its - requests can route to.""" + requests can route to. Display names are read under `label_entries`' ids when given.""" lookup_ids: Final = MappingProxyType(dict(entries)) - return tuple(_catalog_row(row, lookup_ids.get(row["id"], row["id"]), llm_router, team_id) for row in rows) + label_ids: Final = lookup_ids if label_entries is None else MappingProxyType(dict(label_entries)) + return tuple( + _catalog_row( + row, lookup_ids.get(row["id"], row["id"]), label_ids.get(row["id"], row["id"]), llm_router, team_id + ) + for row in rows + ) def codex_model_list_body( @@ -408,9 +415,10 @@ def codex_model_list_body( entries: Sequence[tuple[str, str]], llm_router: Router | None, team_id: str | None = None, + label_entries: Sequence[tuple[str, str]] | None = None, ) -> str: """The `/v1/models?client_version=...` body, logging the models Codex's byte limit left out.""" - body: Final = codex_models_response_json(codex_catalog_rows(rows, entries, llm_router, team_id)) + body: Final = codex_models_response_json(codex_catalog_rows(rows, entries, llm_router, team_id, label_entries)) if body.left_out: verbose_proxy_logger.warning( "Codex model catalog cut at %d bytes, left out: %s. Models offering a service tier are kept first, " diff --git a/litellm/proxy/common_utils/discoverable_model_filter.py b/litellm/proxy/common_utils/discoverable_model_filter.py index d22f1a9f6dc..ca7368b8c72 100644 --- a/litellm/proxy/common_utils/discoverable_model_filter.py +++ b/litellm/proxy/common_utils/discoverable_model_filter.py @@ -109,6 +109,20 @@ def undiscoverable_model_names( ) +def undiscoverable_model_groups( + llm_router: Router | None, + user_api_key_dict: UserAPIKeyAuth, + team_id: str | None, +) -> frozenset[str]: + if llm_router is None or user_api_key_has_admin_view(user_api_key_dict): + return frozenset() + return frozenset( + row["model_name"] + for row in llm_router.get_model_list() or () + if is_undiscoverable_deployment(row) and is_undiscoverable_model_name(row["model_name"], llm_router, team_id) + ) + + def discoverable_rows( rows: Iterable[Mapping[str, object]], user_api_key_dict: UserAPIKeyAuth, diff --git a/litellm/proxy/common_utils/model_listing_utils.py b/litellm/proxy/common_utils/model_listing_utils.py index 0c702ac4139..898b296e83e 100644 --- a/litellm/proxy/common_utils/model_listing_utils.py +++ b/litellm/proxy/common_utils/model_listing_utils.py @@ -21,6 +21,7 @@ from typing import TYPE_CHECKING, Final, cast from pydantic import TypeAdapter, ValidationError import litellm +from litellm.router_utils.common_utils import resolve_model_group_alias if TYPE_CHECKING: from litellm.router import Router @@ -160,13 +161,47 @@ class ClaudeCodeRoutingNames: @dataclass(frozen=True, slots=True) class CallerAliases: - """`own` are the caller's key and team alias maps, the names `/v1/models` lists for it. - `rewrite` are the maps `/chat/completions` rewrites its model through, in the order it - applies them: the team's, the key's in `add_litellm_data_to_request`, then the global - `model_alias_map` and the key's again in `common_processing_pre_call_logic`.""" + """The caller's `team` and `key` alias maps, the names `/v1/models` lists for it, and `router`, + its key's or team's `router_settings.model_group_alias`. + `rewrite` are the maps `/chat/completions` rewrites its model through after auth's `router` + rewrite, in the order it applies them: the team's, the key's in `add_litellm_data_to_request`, + then the global `model_alias_map` and the key's again in `common_processing_pre_call_logic`, + which ends with `router` again.""" - own: tuple[object, ...] - rewrite: tuple[object, ...] + team: object + key: object + router: object = None + + @property + def own(self) -> tuple[object, ...]: + return (self.team, self.key) + + @property + def rewrite(self) -> tuple[object, ...]: + return (self.team, self.key, litellm.model_alias_map, self.key) + + +@dataclass(frozen=True, slots=True) +class DeployedModelGroups: + """Names the router serves `team_id` from deployments of their own, directly or as a + `model_group_alias`, leaving out the `hidden` names, groups `/v1/models` does not list, and, + when a listing callback runs, every name outside `callback_kept`, since it never judged those.""" + + llm_router: Router | None + hidden: Container[str] = frozenset[str]() + team_id: str | None = None + callback_kept: Container[str] | None = None + + def __contains__(self, name: object) -> bool: + if not isinstance(name, str) or self.llm_router is None or name in self.hidden: + return False + if self.callback_kept is not None and name not in self.callback_kept: + return False + group: Final = self.llm_router.routable_model_group(name) + return group not in self.hidden and any( + self.llm_router.deployment_usable_by_team(deployment, self.team_id) + for deployment in self.llm_router.get_all_deployments(model_name=group) + ) def caller_alias_maps( @@ -174,11 +209,12 @@ def caller_alias_maps( team_aliases: object, key_team_id: str | None, listed_team_id: str | None, + router_aliases: object = None, ) -> CallerAliases: """Team aliases count only when listing the team the key authenticated as.""" if listed_team_id is not None and listed_team_id != key_team_id: - return CallerAliases((key_aliases,), (key_aliases, litellm.model_alias_map, key_aliases)) - return CallerAliases((team_aliases, key_aliases), (team_aliases, key_aliases, litellm.model_alias_map, key_aliases)) + return CallerAliases(None, key_aliases, router_aliases) + return CallerAliases(team_aliases, key_aliases, router_aliases) def alias_map(aliases: object) -> Mapping[str, str]: @@ -191,42 +227,148 @@ def alias_map(aliases: object) -> Mapping[str, str]: ) +def _router_alias_map(aliases: object) -> Mapping[str, str]: + try: + entries: Final = _ALIAS_ENTRIES.validate_python(aliases, strict=True) + except ValidationError: + return _NO_ALIASES + targets: Final = ((alias, resolve_model_group_alias(entries, alias)) for alias in entries if isinstance(alias, str)) + return MappingProxyType({alias: target for alias, target in targets if target is not None}) + + +def _hidden_aliases(aliases: object) -> frozenset[str]: + try: + entries: Final = _ALIAS_ENTRIES.validate_python(aliases, strict=True) + except ValidationError: + return frozenset() + return frozenset( + alias + for alias, entry in entries.items() + if isinstance(alias, str) + and isinstance(entry, Mapping) + and cast(Mapping[str, object], entry).get("hidden") is True # cast-ok: isinstance-checked alias entry + ) + + def _alias_names(alias_maps: Sequence[Mapping[str, str]]) -> tuple[str, ...]: return tuple(dict.fromkeys(chain.from_iterable(alias_maps))) -def _rewrite(model_id: str, alias_maps: Sequence[Mapping[str, str]]) -> str | None: - target: Final = reduce(lambda name, aliases: aliases.get(name, name), alias_maps, model_id) - return None if target == model_id else target +@dataclass(frozen=True, slots=True) +class _AliasLayer: + aliases: Mapping[str, str] + checks_target: bool -def alias_target(model_id: str, aliases: CallerAliases, listed: Container[str] = frozenset()) -> str | None: - """The model group `/chat/completions` rewrites `model_id` to, else None. A `model_id` - already `listed` keeps its own row, so it is never rewritten.""" - if model_id in listed: - return None - return _rewrite(model_id, tuple(alias_map(raw) for raw in aliases.rewrite)) +def _alias_layers(aliases: CallerAliases) -> tuple[_AliasLayer, ...]: + """The maps `/chat/completions` rewrites its model through, in order, each marked when auth + checks the caller's access to the model it rewrites to instead of the requested one.""" + router: Final = _AliasLayer(_router_alias_map(aliases.router), checks_target=True) + key: Final = _AliasLayer(alias_map(aliases.key), checks_target=True) + return ( + router, + _AliasLayer(alias_map(aliases.team), checks_target=False), + key, + _AliasLayer(alias_map(litellm.model_alias_map), checks_target=False), + key, + router, + ) + + +def _through_layer(state: tuple[str, bool], layer: _AliasLayer) -> tuple[str, bool]: + name, checks_target = state + target: Final = layer.aliases.get(name, name) + return target, checks_target or (layer.checks_target and target != name) + + +def _dispatch(model_id: str, layers: Sequence[_AliasLayer]) -> tuple[str, bool]: + """The model `/chat/completions` sends `model_id` to, and whether auth checks the caller's + access to that model instead of `model_id`.""" + return reduce(_through_layer, layers, (model_id, False)) + + +def _lookup_id(name: str, lookup_by_response: Mapping[str, str], lookup_ids: Container[str]) -> str | None: + if name in lookup_by_response: + return lookup_by_response[name] + return name if name in lookup_ids else None + + +def _listed_lookup_id( + response_id: str, + lookup_id: str, + layers: Sequence[_AliasLayer], + lookup_by_response: Mapping[str, str], + lookup_ids: Container[str], + deployed: Container[str], +) -> str: + target, checks_target = _dispatch(response_id, layers) + if target == response_id or lookup_id in deployed: + return lookup_id + listed: Final = _lookup_id(target, lookup_by_response, lookup_ids) + if listed is not None: + return listed + return target if not checks_target and target in deployed else lookup_id + + +def _added_lookup_id( + alias: str, layers: Sequence[_AliasLayer], lookup_by_response: Mapping[str, str], lookup_ids: Container[str] +) -> str | None: + target, _ = _dispatch(alias, layers) + return None if target == alias else _lookup_id(target, lookup_by_response, lookup_ids) def alias_listing_entries( entries: Sequence[tuple[str, str]], aliases: CallerAliases, + deployed: Container[str], ) -> tuple[tuple[str, str], ...]: - """`entries` plus one `(alias, lookup_id)` row per key or team alias whose target is - listed. An alias colliding with a listed id keeps the listed entry.""" - maps: Final = tuple(alias_map(raw) for raw in aliases.rewrite) - own: Final = tuple(alias_map(raw) for raw in aliases.own) + """`entries` plus one row per key or team alias whose target is listed. A listed name + with no `deployed` group of its own is looked up under the model `/chat/completions` + sends it to when the caller may call that model through it, so the row describes the + model that answers.""" + layers: Final = _alias_layers(aliases) lookup_by_response: Final = MappingProxyType(dict(entries)) lookup_ids: Final = frozenset(lookup_by_response.values()) - targets: Final = MappingProxyType( - {alias: _rewrite(alias, maps) for alias in _alias_names(own) if alias not in lookup_by_response} + retargeted: Final = tuple( + (response_id, _listed_lookup_id(response_id, lookup_id, layers, lookup_by_response, lookup_ids, deployed)) + for response_id, lookup_id in entries ) added: Final = tuple( - (alias, lookup_by_response.get(target, target)) - for alias, target in targets.items() - if target is not None and (target in lookup_by_response or target in lookup_ids) + (alias, lookup_id) + for alias in _alias_names(tuple(alias_map(raw) for raw in aliases.own)) + if alias not in lookup_by_response + and (lookup_id := _added_lookup_id(alias, layers, lookup_by_response, lookup_ids)) is not None ) - return (*entries, *added) + return (*retargeted, *added) + + +def label_entries(listed: Sequence[tuple[str, str]], entries: Sequence[tuple[str, str]]) -> tuple[tuple[str, str], ...]: + """`entries` with every `listed` name read under its own lookup id again, so an alias + `alias_listing_entries` describes by its target keeps its own display name, the way a + `model_group_alias` does.""" + own: Final = MappingProxyType(dict(listed)) + return tuple((response_id, own.get(response_id, lookup_id)) for response_id, lookup_id in entries) + + +def alias_target( + model_id: str, + aliases: CallerAliases, + entries: Sequence[tuple[str, str]], + deployed: Container[str], +) -> str | None: + """The id `GET /v1/models/{model_id}` looks `model_id` up under when one of the caller's + aliases rewrites it, else None. A listed `model_id` resolves to the row + `alias_listing_entries` lists for it.""" + layers: Final = _alias_layers(aliases) + lookup_by_response: Final = MappingProxyType(dict(entries)) + listed_lookup_id: Final = lookup_by_response.get(model_id) + if listed_lookup_id is None: + target, _ = _dispatch(model_id, layers) + return None if target == model_id or model_id in _hidden_aliases(aliases.router) else target + lookup_id: Final = _listed_lookup_id( + model_id, listed_lookup_id, layers, lookup_by_response, frozenset(lookup_by_response.values()), deployed + ) + return None if lookup_id == listed_lookup_id else lookup_id def claude_code_requested_group( diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 887ccbf180e..03ab1752a92 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -433,7 +433,11 @@ from litellm.proxy.common_utils.config_sync_pubsub import ConfigSyncSubscriber from litellm.proxy.common_utils.credential_hydration import decrypted_or_stored from litellm.proxy.common_utils.debug_utils import init_verbose_loggers from litellm.proxy.common_utils.debug_utils import router as debugging_endpoints_router -from litellm.proxy.common_utils.discoverable_model_filter import discoverable_rows, undiscoverable_model_names +from litellm.proxy.common_utils.discoverable_model_filter import ( + discoverable_rows, + undiscoverable_model_groups, + undiscoverable_model_names, +) from litellm.proxy.common_utils.encrypt_decrypt_utils import ( decrypt_value_helper, encrypt_value_helper, @@ -466,13 +470,14 @@ from litellm.proxy.common_utils.load_config_utils import get_config_from_bucket from litellm.proxy.common_utils.model_deprecation import collect_model_deprecations from litellm.proxy.common_utils.model_listing_utils import ( ClaudeCodeRoutingNames, + DeployedModelGroups, TeamModelNameTranslator, alias_listing_entries, alias_target, caller_alias_maps, claude_code_view_ids, configured_display_names, - is_claude_code_client, + label_entries, ) from litellm.proxy.common_utils.openai_endpoint_utils import ( remove_sensitive_info_from_deployment, @@ -7950,7 +7955,7 @@ class ProxyConfig: user_api_key_dict: UserAPIKeyAuth | None, prisma_client: PrismaClient | None, proxy_logging_obj: ProxyLogging | None = None, - ) -> dict | None: + ) -> Mapping[str, object] | None: return await self._get_hierarchical_router_settings(user_api_key_dict, prisma_client, proxy_logging_obj) async def _get_hierarchical_router_settings( @@ -11878,6 +11883,25 @@ async def _names_hidden_by_listing_callbacks( ) +def _describable_model_groups( + listed_entries: Sequence[tuple[str, str]], + hidden_names: frozenset[str] | set[str], + user_api_key_dict: UserAPIKeyAuth, + team_id: str | None, +) -> DeployedModelGroups: + listing_team_id: Final = team_id or user_api_key_dict.team_id + return DeployedModelGroups( + llm_router, + hidden_names | undiscoverable_model_groups(llm_router, user_api_key_dict, listing_team_id), + listing_team_id, + ( + frozenset(lookup_id for _, lookup_id in listed_entries) + if ProxyLogging.callback_capabilities().listed_models_filters + else None + ), + ) + + async def _entries_kept_by_listing_callbacks( entries: Sequence[tuple[str, str]], user_api_key_dict: UserAPIKeyAuth ) -> tuple[tuple[str, str], ...]: @@ -11965,21 +11989,20 @@ async def model_list( http_request is not None and http_request.headers.get("anthropic-version") is not None ) client_headers: Final[Mapping[str, str]] = http_request.headers if http_request is not None else _EMPTY_HEADERS - view_router_settings: Final = ( - await proxy_config.get_hierarchical_router_settings(user_api_key_dict, prisma_client, proxy_logging_obj) - if wants_anthropic_format and is_claude_code_client(client_headers) - else None - ) - view_aliases: Final = ( - view_router_settings.get("model_group_alias") if isinstance(view_router_settings, Mapping) else None + caller_router_settings: Final = await proxy_config.get_hierarchical_router_settings( + user_api_key_dict, prisma_client, proxy_logging_obj ) caller_aliases: Final = caller_alias_maps( - user_api_key_dict.aliases, user_api_key_dict.team_model_aliases, user_api_key_dict.team_id, team_id + user_api_key_dict.aliases, + user_api_key_dict.team_model_aliases, + user_api_key_dict.team_id, + team_id, + caller_router_settings.get("model_group_alias") if isinstance(caller_router_settings, Mapping) else None, ) routing_names: Final = ClaudeCodeRoutingNames( llm_router, team_id or user_api_key_dict.team_id, - (*caller_aliases.rewrite, view_aliases), + (*caller_aliases.rewrite, caller_aliases.router), ) # Validate scope parameter if provided @@ -12002,7 +12025,7 @@ async def model_list( ) # Compute once — used in both branches below to hide paused models from the listing. - blocked_names: Final = llm_router.get_fully_blocked_model_names() if llm_router is not None else set() + blocked_names: Final = llm_router.get_fully_blocked_model_names() if llm_router is not None else set[str]() # Opt-in: also hide models whose deployments are all unhealthy per background # health checks. Empty when health state is unavailable or stale (fail open). @@ -12119,12 +12142,15 @@ async def model_list( # The internal routing key drives the metadata/fallback lookup, while the # public name is what the client sees as the model id. model_data = [] - entries: Final = alias_listing_entries( - await _entries_kept_by_listing_callbacks( - TeamModelNameTranslator.listing_entries(all_models, llm_router, settings), user_api_key_dict - ), - caller_aliases, + listed_entries: Final = await _entries_kept_by_listing_callbacks( + TeamModelNameTranslator.listing_entries(all_models, llm_router, settings), user_api_key_dict ) + entries: Final = alias_listing_entries( + listed_entries, + caller_aliases, + _describable_model_groups(listed_entries, hidden_names, user_api_key_dict, team_id), + ) + labels: Final = label_entries(listed_entries, entries) for response_id, lookup_id in entries: model_info = create_model_info_response( model_id=lookup_id, @@ -12139,14 +12165,14 @@ async def model_list( listing: Final = cast(Sequence[ModelInfoResponse], model_data) # cast-ok: rows built above if client_version is not None: return Response( - content=codex_model_list_body(listing, entries, llm_router, team_id or user_api_key_dict.team_id), + content=codex_model_list_body(listing, entries, llm_router, team_id or user_api_key_dict.team_id, labels), media_type="application/json", ) if wants_anthropic_format: return create_anthropic_model_list_response( listing, - display_names=configured_display_names(entries, llm_router), + display_names=configured_display_names(labels, llm_router), listed_ids=claude_code_view_ids( listing, client_headers, @@ -12216,7 +12242,7 @@ async def model_info( # Mirror /v1/models' visibility filter so first-occurrence resolution # cannot land on a deployment the listing had hidden. Undiscoverable # models stay retrievable by id, they only drop out of the alias guard. - blocked_names: Final = llm_router.get_fully_blocked_model_names() if llm_router is not None else set() + blocked_names: Final = llm_router.get_fully_blocked_model_names() if llm_router is not None else set[str]() unhealthy_names: Final = await get_hidden_unhealthy_model_names( healthy_only=healthy_only, general_settings=settings, @@ -12224,6 +12250,16 @@ async def model_info( ) hidden_names: Final = blocked_names | unhealthy_names internal_to_public: Final = TeamModelNameTranslator.build_internal_to_public_map(llm_router, settings) + caller_router_settings: Final = await proxy_config.get_hierarchical_router_settings( + user_api_key_dict, prisma_client, proxy_logging_obj + ) + caller_aliases: Final = caller_alias_maps( + user_api_key_dict.aliases, + user_api_key_dict.team_model_aliases, + user_api_key_dict.team_id, + team_id, + caller_router_settings.get("model_group_alias") if isinstance(caller_router_settings, Mapping) else None, + ) callback_hidden_names: Final = await _names_hidden_by_listing_callbacks( user_api_key_dict, tuple( @@ -12241,17 +12277,14 @@ async def model_info( all_models, llm_router, user_api_key_dict, team_id or user_api_key_dict.team_id ) + listed_entries: Final = TeamModelNameTranslator.listing_entries( + tuple(m for m in all_models if m not in undiscoverable_names), llm_router, settings + ) aliased_model_id: Final = alias_target( model_id, - caller_alias_maps( - user_api_key_dict.aliases, user_api_key_dict.team_model_aliases, user_api_key_dict.team_id, team_id - ), - frozenset( - response_id - for response_id, _ in TeamModelNameTranslator.listing_entries( - tuple(m for m in all_models if m not in undiscoverable_names), llm_router, settings - ) - ), + caller_aliases, + listed_entries, + _describable_model_groups(listed_entries, hidden_names, user_api_key_dict, team_id), ) resolved_model_id: Final = TeamModelNameTranslator.resolve_public_name( model_id=aliased_model_id or model_id, @@ -12260,14 +12293,16 @@ async def model_info( general_settings=settings, ) - # Validate that the requested model is accessible - validate_model_access(model_id=resolved_model_id, available_models=all_models) + if all(response_id != model_id for response_id, _ in listed_entries): + validate_model_access(model_id=resolved_model_id, available_models=all_models) # Get provider information from the router deployment if llm_router is None: raise HTTPException(status_code=500, detail="Router not initialized") - deployment: Final = llm_router.get_deployment_by_model_group_name(resolved_model_id) + deployment: Final = llm_router.get_deployment_by_model_group_name( + llm_router.routable_model_group(resolved_model_id) + ) if deployment is None: raise HTTPException( status_code=404, diff --git a/tests/unit/proxy/common_utils/test_model_listing_utils.py b/tests/unit/proxy/common_utils/test_model_listing_utils.py index 97e38d5e17a..3acaa7605a2 100644 --- a/tests/unit/proxy/common_utils/test_model_listing_utils.py +++ b/tests/unit/proxy/common_utils/test_model_listing_utils.py @@ -27,8 +27,8 @@ def _marked(name): return f"{_encoded(name)}[1m]" -def _caller(*maps: object) -> CallerAliases: - return CallerAliases(maps, maps) +def _caller(team: object = None, key: object = None) -> CallerAliases: + return CallerAliases(team, key) def _row(name, limit=1000000): @@ -134,42 +134,87 @@ def test_mutation_breaking_the_hex_name_cannot_route_to_the_source(source): def test_team_alias_is_listed_under_its_target_metadata_and_only_when_the_target_is_accessible() -> None: entries = [("gpt-4.1-mini", "gpt-4.1-mini"), ("team-public", "model_name_team_1_abc")] - aliases = ( + aliases = _caller( {"gpt-4.1-mini": "team-public"}, - None, {"claude-sonnet-4-5": "gpt-4.1-mini", "via-public": "team-public", "not-granted": "gpt-4.1"}, ) - assert alias_listing_entries(entries, _caller(*aliases)) == ( + deployed = frozenset({"gpt-4.1-mini", "model_name_team_1_abc", "gpt-4.1"}) + assert alias_listing_entries(entries, aliases, deployed) == ( *entries, ("claude-sonnet-4-5", "gpt-4.1-mini"), ("via-public", "model_name_team_1_abc"), ) - assert alias_listing_entries(entries, _caller(None, {})) == tuple(entries) + assert alias_listing_entries(entries, _caller(None, {}), deployed) == tuple(entries) def test_alias_target_resolves_the_requested_alias_across_key_and_team_maps() -> None: maps = _caller({"o": "gpt-4.1"}, {"claude-sonnet-4-5": "gpt-4.1-mini"}) - assert alias_target("claude-sonnet-4-5", maps) == "gpt-4.1-mini" - assert alias_target("gpt-4.1-mini", _caller(None, {"claude-sonnet-4-5": "gpt-4.1-mini"})) is None + assert alias_target("claude-sonnet-4-5", maps, (), frozenset()) == "gpt-4.1-mini" + assert alias_target("gpt-4.1-mini", _caller(None, {"claude-sonnet-4-5": "gpt-4.1-mini"}), (), frozenset()) is None -def test_alias_colliding_with_a_listed_id_keeps_the_listed_model_at_list_and_retrieval() -> None: +@pytest.mark.parametrize("layer", ["team", "key", "global"]) +def test_alias_colliding_with_a_deployed_listed_model_keeps_that_model_at_list_and_retrieval( + monkeypatch: pytest.MonkeyPatch, layer: str +) -> None: + alias = {"fast": "gpt-4.1-mini"} + monkeypatch.setattr(litellm, "model_alias_map", alias if layer == "global" else {}) + maps = _caller(alias if layer == "team" else None, alias if layer == "key" else None) entries = [("fast", "fast"), ("gpt-4.1-mini", "gpt-4.1-mini")] - maps = _caller({"fast": "gpt-4.1-mini"}) - listed = frozenset(response_id for response_id, _ in entries) + deployed = frozenset({"fast", "gpt-4.1-mini"}) - assert alias_listing_entries(entries, maps) == tuple(entries) - assert alias_target("fast", maps, listed) is None - assert alias_target("fast", maps) == "gpt-4.1-mini" + assert alias_listing_entries(entries, maps, deployed) == tuple(entries) + assert alias_target("fast", maps, entries, deployed) is None + assert alias_target("fast", maps, (), deployed) == "gpt-4.1-mini" + + +def test_listed_alias_is_looked_up_under_the_listed_model_it_rewrites_to_at_list_and_retrieval() -> None: + entries = [ + ("fast", "fast"), + ("gpt-4.1-mini", "gpt-4.1-mini"), + ("chat", "chat"), + ("internal", "internal"), + ("team-public", "model_name_team_1_abc"), + ] + maps = _caller({"fast": "gpt-4.1-mini", "chat": "team-public", "internal": "model_name_team_1_abc"}) + deployed = frozenset({"gpt-4.1-mini", "model_name_team_1_abc"}) + listing = alias_listing_entries(entries, maps, deployed) + + assert listing == ( + ("fast", "gpt-4.1-mini"), + ("gpt-4.1-mini", "gpt-4.1-mini"), + ("chat", "model_name_team_1_abc"), + ("internal", "model_name_team_1_abc"), + ("team-public", "model_name_team_1_abc"), + ) + assert all( + (alias_target(response_id, maps, entries, deployed) or dict(entries)[response_id]) == lookup_id + for response_id, lookup_id in listing + ) + + +def test_listed_team_alias_reports_its_deployed_target_unless_a_key_alias_needs_the_caller_to_list_it() -> None: + entries = [("fast", "fast"), ("gpt-4.1-mini", "gpt-4.1-mini")] + deployed = frozenset({"gpt-4.1-mini", "gpt-4.1"}) + through_team = _caller({"fast": "gpt-4.1"}) + through_key = _caller(None, {"fast": "gpt-4.1"}) + to_nothing_deployed = _caller({"fast": "restricted"}) + + assert alias_listing_entries(entries, through_team, deployed) == (("fast", "gpt-4.1"), entries[1]) + assert alias_target("fast", through_team, entries, deployed) == "gpt-4.1" + for maps in (through_key, to_nothing_deployed): + assert alias_listing_entries(entries, maps, deployed) == tuple(entries) + assert alias_target("fast", maps, entries, deployed) is None def test_alias_maps_apply_in_the_order_chat_completions_applies_them() -> None: team_then_key = _caller({"fast": "gpt-4.1-mini", "hop": "mid"}, {"fast": "gpt-4.1", "mid": "gpt-4.1"}) entries = [("gpt-4.1-mini", "gpt-4.1-mini"), ("gpt-4.1", "gpt-4.1")] + deployed = frozenset({"gpt-4.1-mini", "gpt-4.1"}) - assert alias_target("fast", team_then_key) == "gpt-4.1-mini" - assert alias_target("hop", team_then_key) == "gpt-4.1" - assert alias_listing_entries(entries, team_then_key) == ( + assert alias_target("fast", team_then_key, (), deployed) == "gpt-4.1-mini" + assert alias_target("hop", team_then_key, (), deployed) == "gpt-4.1" + assert alias_listing_entries(entries, team_then_key, deployed) == ( *entries, ("fast", "gpt-4.1-mini"), ("hop", "gpt-4.1"), @@ -180,17 +225,20 @@ def test_alias_maps_apply_in_the_order_chat_completions_applies_them() -> None: def test_one_bad_alias_entry_hides_only_itself() -> None: aliases = {"fast": "gpt-4.1-mini", "broken": 5, 7: "gpt-4.1-mini"} entries = [("gpt-4.1-mini", "gpt-4.1-mini")] + deployed = frozenset({"gpt-4.1-mini"}) - assert alias_listing_entries(entries, _caller(aliases)) == (*entries, ("fast", "gpt-4.1-mini")) - assert alias_target("fast", _caller(aliases)) == "gpt-4.1-mini" + assert alias_listing_entries(entries, _caller(aliases), deployed) == (*entries, ("fast", "gpt-4.1-mini")) + assert alias_target("fast", _caller(aliases), (), deployed) == "gpt-4.1-mini" def test_chained_key_alias_is_listed_only_when_its_final_target_is_listable() -> None: key_aliases = {"a": "b", "b": "hidden"} entries = [("b", "b")] + maps = caller_alias_maps(key_aliases, None, "team-a", None) + deployed = frozenset({"b", "hidden"}) - assert alias_listing_entries(entries, caller_alias_maps(key_aliases, None, "team-a", None)) == (*entries,) - assert alias_target("a", caller_alias_maps(key_aliases, None, "team-a", None)) == "hidden" + assert alias_listing_entries(entries, maps, deployed) == (*entries,) + assert alias_target("a", maps, (), deployed) == "hidden" def test_team_aliases_only_apply_when_listing_the_team_the_key_authenticated_as( @@ -198,12 +246,10 @@ def test_team_aliases_only_apply_when_listing_the_team_the_key_authenticated_as( ) -> None: key_aliases, team_aliases, global_aliases = {"k": "gpt-4.1"}, {"t": "gpt-4.1-mini"}, {"g": "gpt-4.1"} monkeypatch.setattr(litellm, "model_alias_map", global_aliases) - own_team = CallerAliases((team_aliases, key_aliases), (team_aliases, key_aliases, global_aliases, key_aliases)) - assert caller_alias_maps(key_aliases, team_aliases, "team-a", None) == own_team + own_team = caller_alias_maps(key_aliases, team_aliases, "team-a", None) + assert own_team.rewrite == (team_aliases, key_aliases, global_aliases, key_aliases) assert caller_alias_maps(key_aliases, team_aliases, "team-a", "team-a") == own_team - assert caller_alias_maps(key_aliases, team_aliases, "team-a", "team-b") == CallerAliases( - (key_aliases,), (key_aliases, global_aliases, key_aliases) - ) + assert caller_alias_maps(key_aliases, team_aliases, "team-a", "team-b") == CallerAliases(None, key_aliases) def test_global_alias_rewrites_between_the_two_key_passes_like_chat_completions( @@ -213,18 +259,20 @@ def test_global_alias_rewrites_between_the_two_key_passes_like_chat_completions( key_aliases = {"a": "b", "b": "c"} entries = [("c", "c"), ("d", "d")] maps = caller_alias_maps(key_aliases, None, "team-a", None) + deployed = frozenset({"c", "d"}) - assert alias_target("a", maps) == "d" - assert alias_listing_entries(entries, maps) == (*entries, ("a", "d"), ("b", "c")) + assert alias_target("a", maps, (), deployed) == "d" + assert alias_listing_entries(entries, maps, deployed) == (*entries, ("a", "d"), ("b", "c")) def test_global_aliases_rewrite_but_are_not_listed_as_caller_rows(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(litellm, "model_alias_map", {"g": "gpt-4.1-mini"}) entries = [("gpt-4.1-mini", "gpt-4.1-mini")] maps = caller_alias_maps({"k": "g"}, None, "team-a", None) + deployed = frozenset({"gpt-4.1-mini"}) - assert alias_listing_entries(entries, maps) == (*entries, ("k", "gpt-4.1-mini")) - assert alias_target("g", maps) == "gpt-4.1-mini" + assert alias_listing_entries(entries, maps, deployed) == (*entries, ("k", "gpt-4.1-mini")) + assert alias_target("g", maps, (), deployed) == "gpt-4.1-mini" def test_team_public_name_uses_the_same_scope_at_list_and_request(): diff --git a/tests/unit/proxy/proxy_server/test_team_model_name_translation.py b/tests/unit/proxy/proxy_server/test_team_model_name_translation.py index b848c4f1976..679959b5728 100644 --- a/tests/unit/proxy/proxy_server/test_team_model_name_translation.py +++ b/tests/unit/proxy/proxy_server/test_team_model_name_translation.py @@ -1414,6 +1414,7 @@ def _public_named_router(*team_rows: dict) -> MagicMock: router = MagicMock() router.get_model_list.return_value = list(team_rows) router.get_model_listing_info.return_value = None + router.routable_model_group.side_effect = lambda model_name: model_name return router diff --git a/tests/unit/proxy/test_model_list_aliases.py b/tests/unit/proxy/test_model_list_aliases.py index 25a941bf2ae..0ec35a82e8d 100644 --- a/tests/unit/proxy/test_model_list_aliases.py +++ b/tests/unit/proxy/test_model_list_aliases.py @@ -4,12 +4,19 @@ Tests for key and team `model_aliases` on the model listing endpoints: GET /v1/m An alias the caller can complete on is listed next to its target and resolves by name. """ +import json +from collections.abc import Sequence + import pytest +from fastapi import HTTPException from starlette.requests import Request +import litellm from litellm import Router +from litellm.integrations.custom_logger import CustomLogger from litellm.proxy import proxy_server from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.utils import ProxyLogging def _deployment(model_name: str, model: str = "openai/gpt-4.1-mini", **model_info: str | bool) -> dict[str, object]: @@ -20,15 +27,41 @@ def _deployment(model_name: str, model: str = "openai/gpt-4.1-mini", **model_inf } +class _ListingGate(CustomLogger): + def __init__(self, hidden: frozenset[str]) -> None: + super().__init__() + self.hidden = hidden + + async def async_filter_listed_models( + self, user_api_key_dict: UserAPIKeyAuth, model_names: Sequence[str] + ) -> Sequence[str]: + return [name for name in model_names if name not in self.hidden] + + +class _ListingAllowList(CustomLogger): + def __init__(self, kept: frozenset[str]) -> None: + super().__init__() + self.kept = kept + + async def async_filter_listed_models( + self, user_api_key_dict: UserAPIKeyAuth, model_names: Sequence[str] + ) -> Sequence[str]: + return [name for name in model_names if name in self.kept] + + @pytest.fixture def router(monkeypatch: pytest.MonkeyPatch) -> Router: router = Router( model_list=[ _deployment("gpt-4.1-mini"), - _deployment("gpt-4.1", model="openai/gpt-4.1"), - _deployment("model_name_team1_abc", team_id="team1", team_public_model_name="team-chat"), + _deployment("gpt-4.1", model="openai/gpt-4.1", display_name="GPT 4.1"), + _deployment( + "model_name_team1_abc", model="openai/gpt-4o", team_id="team1", team_public_model_name="team-chat" + ), _deployment("hidden", model="anthropic/claude-sonnet-4-5", discoverable=False), - ] + _deployment("paused", model="openai/gpt-4.1", blocked=True), + ], + model_group_alias={"legacy": "gpt-4.1"}, ) monkeypatch.setattr(proxy_server, "llm_router", router) monkeypatch.setattr(proxy_server, "llm_model_list", router.model_list) @@ -149,3 +182,171 @@ async def test_v1_models_by_id_keeps_the_alias_as_id_when_it_targets_a_team_scop assert "chat" in await _v1_models(caller) response = await proxy_server.model_info(model_id="chat", user_api_key_dict=caller) assert response["id"] == "chat" + + +@pytest.mark.asyncio +async def test_alias_the_key_also_lists_reports_its_target_metadata_at_list_and_retrieval(router: Router) -> None: + caller = _team_member( + models=["gpt-4.1-mini", "model_name_team1_abc", "fast"], team_model_aliases={"fast": "gpt-4.1-mini"} + ) + + response = await proxy_server.model_list(user_api_key_dict=caller, request=None) + rows = {row["id"]: row for row in response["data"]} + assert rows["fast"] == {**rows["gpt-4.1-mini"], "id": "fast"} + target = await proxy_server.model_info(model_id="gpt-4.1-mini", user_api_key_dict=caller) + assert await proxy_server.model_info(model_id="fast", user_api_key_dict=caller) == {**target, "id": "fast"} + + +def _keyholder(*models: str) -> UserAPIKeyAuth: + return UserAPIKeyAuth(api_key="sk-test", user_id="u", user_role=LitellmUserRoles.INTERNAL_USER, models=list(models)) + + +@pytest.mark.asyncio +async def test_team_alias_to_a_model_the_team_does_not_list_reports_that_model_at_list_and_retrieval( + router: Router, +) -> None: + caller = _team_member(models=["gpt-4.1-mini", "model_name_team1_abc", "fast"], team_model_aliases={"fast": "gpt-4.1"}) + target_holder = _keyholder("gpt-4.1") + + rows = {row["id"]: row for row in (await proxy_server.model_list(user_api_key_dict=caller, request=None))["data"]} + target_rows = (await proxy_server.model_list(user_api_key_dict=target_holder, request=None))["data"] + assert [rows["fast"]] == [{**row, "id": "fast"} for row in target_rows if row["id"] == "gpt-4.1"] + target = await proxy_server.model_info(model_id="gpt-4.1", user_api_key_dict=target_holder) + assert await proxy_server.model_info(model_id="fast", user_api_key_dict=caller) == {**target, "id": "fast"} + + +@pytest.mark.asyncio +async def test_alias_named_like_a_listed_model_keeps_that_model_s_metadata_at_list_and_retrieval( + router: Router, +) -> None: + plain = await proxy_server.model_list(user_api_key_dict=_team_member(), request=None) + shadowing = _team_member(team_model_aliases={"gpt-4.1-mini": "team-chat"}) + + assert (await proxy_server.model_list(user_api_key_dict=shadowing, request=None))["data"] == plain["data"] + assert await proxy_server.model_info(model_id="gpt-4.1-mini", user_api_key_dict=shadowing) == ( + await proxy_server.model_info(model_id="gpt-4.1-mini", user_api_key_dict=_team_member()) + ) + + +@pytest.mark.asyncio +async def test_router_alias_the_caller_lists_resolves_at_retrieval_like_its_listing_row(router: Router) -> None: + caller = _keyholder("gpt-4.1", "legacy") + + rows = {row["id"]: row for row in (await proxy_server.model_list(user_api_key_dict=caller, request=None))["data"]} + assert rows["legacy"] == {**rows["gpt-4.1"], "id": "legacy"} + target = await proxy_server.model_info(model_id="gpt-4.1", user_api_key_dict=caller) + assert await proxy_server.model_info(model_id="legacy", user_api_key_dict=caller) == {**target, "id": "legacy"} + + +def _alias_holder(**aliases: dict[str, object]) -> UserAPIKeyAuth: + return UserAPIKeyAuth( + api_key="sk-test", + user_id="u", + user_role=LitellmUserRoles.INTERNAL_USER, + models=["gpt-4.1", "fast"], + **aliases, + ) + + +@pytest.mark.asyncio +async def test_listed_alias_keeps_its_own_display_name_in_the_anthropic_and_codex_listings(router: Router) -> None: + caller = _alias_holder(aliases={"fast": "gpt-4.1"}) + + anthropic = await proxy_server.model_list(user_api_key_dict=caller, request=_anthropic_request()) + assert {row["id"]: row["display_name"] for row in anthropic["data"]} == {"gpt-4.1": "GPT 4.1", "fast": "fast"} + codex = await proxy_server.model_list(user_api_key_dict=caller, request=None, client_version="0.200.0") + target, alias = json.loads(codex.body)["models"] + assert (target["display_name"], alias["display_name"]) == ("GPT 4.1", "fast") + assert {**alias, "slug": "gpt-4.1", "display_name": "GPT 4.1", "priority": 0} == target + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "model_group_alias", [{"fast": "gpt-4.1"}, {"fast": {"model": "gpt-4.1", "hidden": True}}] +) +async def test_key_router_settings_alias_the_key_lists_reports_its_target_at_list_and_retrieval( + router: Router, model_group_alias: dict[str, object] +) -> None: + caller = _alias_holder(router_settings={"model_group_alias": model_group_alias}) + + rows = {row["id"]: row for row in (await proxy_server.model_list(user_api_key_dict=caller, request=None))["data"]} + assert rows["fast"] == {**rows["gpt-4.1"], "id": "fast"} + target = await proxy_server.model_info(model_id="gpt-4.1", user_api_key_dict=caller) + assert await proxy_server.model_info(model_id="fast", user_api_key_dict=caller) == {**target, "id": "fast"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("target", "team_id"), [("paused", "team1"), ("hidden", "team1"), ("model_name_team1_abc", "team2")] +) +async def test_alias_to_a_model_the_listing_hides_is_not_described_by_it( + router: Router, target: str, team_id: str +) -> None: + models = ["gpt-4.1-mini", "fast"] + caller = _team_member(team_id=team_id, models=models, team_model_aliases={"fast": target}) + + listing = await proxy_server.model_list(user_api_key_dict=caller, request=None, include_metadata=True) + unaliased = await proxy_server.model_list( + user_api_key_dict=_team_member(team_id=team_id, models=models), request=None, include_metadata=True + ) + assert listing["data"] == unaliased["data"] + with pytest.raises(HTTPException) as retrieval: + await proxy_server.model_info(model_id="fast", user_api_key_dict=caller) + assert retrieval.value.status_code == 404 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("target", "gated", "models"), + [ + ("gpt-4.1", "gpt-4.1", ["gpt-4.1-mini", "fast"]), + ("gpt-4.1", "gpt-4.1", ["gpt-4.1-mini", "fast", "gpt-4.1"]), + ("legacy", "legacy", ["gpt-4.1-mini", "fast"]), + ("legacy", "gpt-4.1", ["gpt-4.1-mini", "fast"]), + ("model_name_team1_abc", "team-chat", ["gpt-4.1-mini", "fast", "model_name_team1_abc"]), + ], +) +async def test_alias_to_a_model_a_listing_callback_hides_is_not_described_by_it( + router: Router, monkeypatch: pytest.MonkeyPatch, target: str, gated: str, models: list[str] +) -> None: + monkeypatch.setattr(litellm, "callbacks", [_ListingGate(frozenset({gated}))]) + ProxyLogging._callback_capabilities_cache.clear() + caller = _team_member(models=models, team_model_aliases={"fast": target}) + + listing = await proxy_server.model_list(user_api_key_dict=caller, request=None, include_metadata=True) + unaliased = await proxy_server.model_list( + user_api_key_dict=_team_member(models=models), request=None, include_metadata=True + ) + assert listing["data"] == unaliased["data"] + with pytest.raises(HTTPException) as retrieval: + await proxy_server.model_info(model_id="fast", user_api_key_dict=caller) + assert retrieval.value.status_code == 404 + + +@pytest.mark.asyncio +async def test_allow_list_callback_keeps_every_name_it_allows_when_an_alias_points_past_them( + router: Router, monkeypatch: pytest.MonkeyPatch +) -> None: + caller = _team_member(models=["gpt-4.1-mini", "legacy", "fast"], team_model_aliases={"fast": "gpt-4.1"}) + listed = await _v1_models(caller) + + monkeypatch.setattr(litellm, "callbacks", [_ListingAllowList(frozenset(listed))]) + ProxyLogging._callback_capabilities_cache.clear() + assert await _v1_models(caller) == listed + + +@pytest.mark.asyncio +async def test_hidden_router_settings_alias_the_key_does_not_list_is_not_retrievable(router: Router) -> None: + caller = UserAPIKeyAuth( + api_key="sk-test", + user_id="u", + user_role=LitellmUserRoles.INTERNAL_USER, + models=["gpt-4.1"], + router_settings={"model_group_alias": {"fast": "gpt-4.1", "quiet": {"model": "gpt-4.1", "hidden": True}}}, + ) + + target = await proxy_server.model_info(model_id="gpt-4.1", user_api_key_dict=caller) + assert await proxy_server.model_info(model_id="fast", user_api_key_dict=caller) == {**target, "id": "fast"} + with pytest.raises(HTTPException) as retrieval: + await proxy_server.model_info(model_id="quiet", user_api_key_dict=caller) + assert retrieval.value.status_code == 404