diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py index 16a68bc52e0..f17bb827641 100644 --- a/litellm/integrations/custom_logger.py +++ b/litellm/integrations/custom_logger.py @@ -440,7 +440,12 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac """Runs on the model listing routes (`/v1/models`, `/v1/models/{id}`, `/model/info`, `/model_group/info`) with the public model names the route would otherwise return, so a lookup of one model may offer just that name: decide per name, never by position in the - sequence. Return the names to keep as a sequence of strings; a name left out disappears + sequence. The names also include each unlisted model that a key alias, a team alias, or + `litellm.model_alias_map` sends a listed name to without checking the caller's access to that + model, and the model a `model_group_alias` of that name resolves to, and the listed name's row + carries that model's info only while the callback keeps all of them. Leaving one out hides + no listed name. Alias names a key or team maps without listing them are not offered. Return + the names to keep as a sequence of strings; a name left out disappears from every listing, any alias of it offered in the same call goes with it, and `/v1/models/{id}` answers 404 for it, exactly as for a model that does not exist. Names outside `model_names` are ignored, so a callback can only narrow the listing, never widen diff --git a/litellm/proxy/common_utils/model_listing_utils.py b/litellm/proxy/common_utils/model_listing_utils.py index 86b444663f0..4381a533fd4 100644 --- a/litellm/proxy/common_utils/model_listing_utils.py +++ b/litellm/proxy/common_utils/model_listing_utils.py @@ -352,6 +352,18 @@ def alias_listing_entries( return (*retargeted, *added) +def unlisted_alias_targets(entries: Sequence[tuple[str, str]], aliases: CallerAliases) -> tuple[str, ...]: + layers: Final = _alias_layers(aliases) + listed: Final = frozenset(chain.from_iterable(entries)) + return tuple( + dict.fromkeys( + target + for target, checks_target in (_dispatch(response_id, layers) for response_id, _ in entries) + if not checks_target and target not in listed + ) + ) + + def label_entries(listed: Sequence[tuple[str, str]], entries: Sequence[tuple[str, str]]) -> tuple[tuple[str, str], ...]: """`entries` with every `listed` name read under its own lookup id again, so an alias `alias_listing_entries` describes by its target keeps its own display name, the way a @@ -448,7 +460,7 @@ class TeamModelNameTranslator: @staticmethod def _response_to_lookup_map( model_names: Sequence[str], - internal_to_public: dict[str, str], + internal_to_public: Mapping[str, str], ) -> dict[str, str]: """Map each public response id to the first internal lookup id seen in `model_names`, preserving first-occurrence order. First-wins keeps list @@ -484,6 +496,12 @@ class TeamModelNameTranslator: return [(name, name) for name in model_names] return list(TeamModelNameTranslator._response_to_lookup_map(model_names, internal_to_public).items()) + @staticmethod + def public_name_entries( + model_names: Sequence[str], internal_to_public: Mapping[str, str] + ) -> tuple[tuple[str, str], ...]: + return tuple(TeamModelNameTranslator._response_to_lookup_map(model_names, internal_to_public).items()) + @staticmethod def translate_listing( model_names: list[str], diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index eda4614bdf3..9ff37693551 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -582,7 +582,9 @@ from litellm.proxy.common_utils.model_listing_utils import ( caller_alias_maps, claude_code_view_ids, configured_display_names, + dispatched_model, label_entries, + unlisted_alias_targets, ) from litellm.proxy.common_utils.openai_endpoint_utils import ( remove_sensitive_info_from_deployment, @@ -12768,7 +12770,10 @@ class ProxyStartupEvent: async def _names_hidden_by_listing_callbacks( user_api_key_dict: UserAPIKeyAuth, model_names: Sequence[str] ) -> frozenset[str]: - hidden: Final = await proxy_logging_obj.hidden_by_listing_callbacks(user_api_key_dict, model_names) + return _with_router_aliases_to(await proxy_logging_obj.hidden_by_listing_callbacks(user_api_key_dict, model_names)) + + +def _with_router_aliases_to(hidden: frozenset[str]) -> frozenset[str]: if not hidden or llm_router is None: return hidden aliases: Final = llm_router.registry().aliases() @@ -12781,11 +12786,47 @@ async def _names_hidden_by_listing_callbacks( ) +@dataclass(frozen=True, slots=True) +class _ListingCallbackVerdict: + hid: frozenset[str] + hidden: frozenset[str] + kept_alias_targets: frozenset[str] + + +async def _listing_callback_verdict( + user_api_key_dict: UserAPIKeyAuth, + listed_names: Sequence[str], + alias_targets: Sequence[str], + internal_to_public: Mapping[str, str], +) -> _ListingCallbackVerdict: + router_aliases: Final = llm_router.registry().aliases() if llm_router is not None else {} + target_reads: Final = MappingProxyType( + { + target: frozenset( + internal_to_public.get(name, name) + for name in (target, resolve_model_group_alias(router_aliases, target)) + if name is not None + ) + for target in alias_targets + } + ) + offered_hid: Final = await proxy_logging_obj.hidden_by_listing_callbacks( + user_api_key_dict, tuple(dict.fromkeys((*listed_names, *chain.from_iterable(target_reads.values())))) + ) + hid: Final = offered_hid.intersection(listed_names) + return _ListingCallbackVerdict( + hid, + _with_router_aliases_to(hid), + frozenset(target for target, names in target_reads.items() if offered_hid.isdisjoint(names)), + ) + + def _describable_model_groups( listed_entries: Sequence[tuple[str, str]], hidden_names: frozenset[str] | set[str], user_api_key_dict: UserAPIKeyAuth, team_id: str | None, + kept_alias_targets: frozenset[str], ) -> DeployedModelGroups: listing_team_id: Final = team_id or user_api_key_dict.team_id return DeployedModelGroups( @@ -12793,8 +12834,8 @@ def _describable_model_groups( hidden_names | undiscoverable_model_groups(llm_router, user_api_key_dict, listing_team_id), listing_team_id, ( - frozenset(lookup_id for _, lookup_id in listed_entries) - if ProxyLogging.callback_capabilities().listed_models_filters + frozenset((*(lookup_id for _, lookup_id in listed_entries), *kept_alias_targets)) + if ProxyLogging.filters_listed_models() else None ), ) @@ -12896,16 +12937,26 @@ async def _key_model_listing( excluded: Final = hidden_names | undiscoverable_model_names( names, llm_router, user_api_key_dict, team_id or user_api_key_dict.team_id ) - listed_entries: Final = await _entries_kept_by_listing_callbacks( - TeamModelNameTranslator.listing_entries( - [name for name in names if name not in excluded], llm_router, general_settings_view() - ), - user_api_key_dict, + translated_entries: Final = TeamModelNameTranslator.listing_entries( + [name for name in names if name not in excluded], llm_router, general_settings_view() ) - entries: Final = alias_listing_entries( - listed_entries, - caller_aliases, - _describable_model_groups(listed_entries, hidden_names, user_api_key_dict, team_id), + verdict: Final = await _listing_callback_verdict( + user_api_key_dict, + tuple(response_id for response_id, _ in translated_entries), + unlisted_alias_targets(translated_entries, caller_aliases), + TeamModelNameTranslator.build_internal_to_public_map(llm_router, general_settings_view()), + ) + listed_entries: Final = tuple(entry for entry in translated_entries if entry[0] not in verdict.hidden) + entries: Final = tuple( + entry + for entry in alias_listing_entries( + listed_entries, + caller_aliases, + _describable_model_groups( + listed_entries, hidden_names, user_api_key_dict, team_id, verdict.kept_alias_targets + ), + ) + if entry[0] not in verdict.hid ) return _KeyModelListing( rows=tuple( @@ -13248,18 +13299,18 @@ async def model_info( hidden_names: Final = blocked_names | unhealthy_names internal_to_public: Final = TeamModelNameTranslator.build_internal_to_public_map(llm_router, settings) caller_aliases: Final = await request_caller_aliases(user_api_key_dict, team_id) - callback_hidden_names: Final = await _names_hidden_by_listing_callbacks( - user_api_key_dict, - tuple( - response_id - for response_id, _ in TeamModelNameTranslator.listing_entries( - tuple(m for m in all_models if m not in hidden_names), llm_router, settings - ) - ), + offered_entries: Final = TeamModelNameTranslator.listing_entries( + tuple(m for m in all_models if m not in hidden_names), llm_router, settings ) - if hidden_names or callback_hidden_names: + verdict: Final = await _listing_callback_verdict( + user_api_key_dict, + tuple(response_id for response_id, _ in offered_entries), + unlisted_alias_targets(offered_entries, caller_aliases), + internal_to_public, + ) + if hidden_names or verdict.hidden: all_models = [ - m for m in all_models if m not in hidden_names and internal_to_public.get(m, m) not in callback_hidden_names + m for m in all_models if m not in hidden_names and internal_to_public.get(m, m) not in verdict.hidden ] undiscoverable_names: Final = undiscoverable_model_names( all_models, llm_router, user_api_key_dict, team_id or user_api_key_dict.team_id @@ -13268,11 +13319,17 @@ async def model_info( listed_entries: Final = TeamModelNameTranslator.listing_entries( tuple(m for m in all_models if m not in undiscoverable_names), llm_router, settings ) - aliased_model_id: Final = alias_target( - model_id, - caller_aliases, - listed_entries, - _describable_model_groups(listed_entries, hidden_names, user_api_key_dict, team_id), + aliased_model_id: Final = ( + None + if model_id in verdict.hid + else alias_target( + model_id, + caller_aliases, + listed_entries, + _describable_model_groups( + listed_entries, hidden_names, user_api_key_dict, team_id, verdict.kept_alias_targets + ), + ) ) resolved_model_id: Final = TeamModelNameTranslator.resolve_public_name( model_id=aliased_model_id or model_id, @@ -19314,39 +19371,123 @@ async def model_group_info( undiscoverable_group_names: Final = undiscoverable_model_names( all_models_str, llm_router, user_api_key_dict, user_api_key_dict.team_id ) - listed_group_names: Final = tuple(name for name in all_models_str if name not in undiscoverable_group_names) + caller_aliases: Final = await request_caller_aliases(user_api_key_dict) + requested_names: Final = ( + None + if model_group is None + else _names_a_model_group_row_reads(model_group, caller_aliases, llm_router.registry().aliases()) + ) + internal_to_public: Final = TeamModelNameTranslator.build_internal_to_public_map(llm_router, general_settings) + listed_group_names: Final = tuple( + dict.fromkeys( + name + for name in all_models_str + if name not in undiscoverable_group_names + and (requested_names is None or not requested_names.isdisjoint((name, internal_to_public.get(name, name)))) + ) + ) - # Append A2A agents to model groups from litellm.proxy.agent_endpoints.model_list_helpers import ( append_agents_to_model_group, ) - model_groups: Final = await append_agents_to_model_group( - model_groups=get_model_group_info( - llm_router=llm_router, - all_models_str=listed_group_names, - model_group=model_group, - team_id=user_api_key_dict.team_id, + agent_groups: Final = await append_agents_to_model_group(model_groups=[], user_api_key_dict=user_api_key_dict) + verdict: Final = await _listing_callback_verdict( + user_api_key_dict, + tuple( + dict.fromkeys( + ( + *(internal_to_public.get(name, name) for name in listed_group_names), + *(group.model_group for group in agent_groups), + ) + ) ), - user_api_key_dict=user_api_key_dict, + unlisted_alias_targets( + TeamModelNameTranslator.public_name_entries(listed_group_names, internal_to_public), caller_aliases + ), + internal_to_public, ) - internal_to_public: Final = TeamModelNameTranslator.build_internal_to_public_map(llm_router, general_settings) - public_group_names: Final = tuple( - internal_to_public.get(group.model_group, group.model_group) for group in model_groups + kept_entries: Final = tuple( + (name, name) for name in listed_group_names if internal_to_public.get(name, name) not in verdict.hidden ) - callback_hidden_names: Final = await _names_hidden_by_listing_callbacks( - user_api_key_dict, tuple(dict.fromkeys(public_group_names)) + blocked_names: Final = llm_router.health().fully_blocked_model_names() + requested_entries: Final = tuple( + entry + for entry in _listing_with_aliases_to_unpaused_models( + kept_entries, + TeamModelNameTranslator.public_name_entries( + tuple(name for name, _ in kept_entries if name not in blocked_names), internal_to_public + ), + caller_aliases, + _describable_model_groups(kept_entries, blocked_names, user_api_key_dict, None, verdict.kept_alias_targets), + ) + if (model_group is None or entry[0] == model_group) and entry[0] not in verdict.hid + ) + lookup_ids: Final = tuple(dict.fromkeys(lookup_id for _, lookup_id in requested_entries)) + groups_by_lookup_id: Final = MappingProxyType( + dict( + zip( + lookup_ids, + get_model_group_info( + llm_router=llm_router, + all_models_str=lookup_ids, + model_group=None, + team_id=user_api_key_dict.team_id, + ), + strict=True, + ) + ) ) return { "data": [ - group - for group, public_name in zip(model_groups, public_group_names, strict=True) - if public_name not in callback_hidden_names + *( + _model_group_row_listed_as(groups_by_lookup_id[lookup_id], listed_name, lookup_id) + for listed_name, lookup_id in requested_entries + ), + *(group for group in agent_groups if group.model_group not in verdict.hidden), ] } +def _names_a_model_group_row_reads(model_group: str, aliases: CallerAliases, router_aliases: object) -> frozenset[str]: + return frozenset( + name + for name in ( + model_group, + dispatched_model(model_group, aliases), + resolve_model_group_alias(router_aliases, model_group), + ) + if name is not None + ) + + +def _listing_with_aliases_to_unpaused_models( + listed_entries: Sequence[tuple[str, str]], + unpaused_public_entries: Sequence[tuple[str, str]], + aliases: CallerAliases, + deployed: Container[str], +) -> tuple[tuple[str, str], ...]: + described: Final = alias_listing_entries(unpaused_public_entries, aliases, deployed) + lookup_by_name: Final = MappingProxyType(dict(described)) + shown_names: Final = frozenset(name for name, _ in (*listed_entries, *unpaused_public_entries)) + return ( + *((name, lookup_by_name.get(name, lookup_id)) for name, lookup_id in listed_entries), + *(entry for entry in described if entry[0] not in shown_names), + ) + + +def _model_group_row_listed_as(row: ModelGroupInfoProxy, listed_name: str, lookup_id: str) -> ModelGroupInfoProxy: + if listed_name == lookup_id: + return row + return row.model_copy( + update={ + "model_group": listed_name, + "is_public_model_group": listed_name in (litellm.public_model_groups or ()), + } + ) + + @router.get( "/model/settings", description="Returns provider name, description, and required parameters for each provider", diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 47fb9f9d375..0259363616a 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -4977,6 +4977,10 @@ class ProxyLogging: if result is not None: merged_headers.update(result) + @staticmethod + def filters_listed_models() -> bool: + return bool(hook_subscribers(Hook.list_models, ProxyLogging.callback_capabilities().hook_callbacks).entries) + async def hidden_by_listing_callbacks( self, user_api_key_dict: UserAPIKeyAuth, model_names: Sequence[str] ) -> frozenset[str]: diff --git a/tests/unit/proxy/test_model_list_aliases.py b/tests/unit/proxy/test_model_list_aliases.py index 0ec35a82e8d..5a5b17192f3 100644 --- a/tests/unit/proxy/test_model_list_aliases.py +++ b/tests/unit/proxy/test_model_list_aliases.py @@ -1,19 +1,25 @@ """ Tests for key and team `model_aliases` on the model listing endpoints: GET /v1/models -(`model_list`, OpenAI and Anthropic shapes) and GET /v1/models/{id} (`model_info`). -An alias the caller can complete on is listed next to its target and resolves by name. +(`model_list`, OpenAI and Anthropic shapes), GET /v1/models/{id} (`model_info`) and +GET /model_group/info (`model_group_info`). An alias the caller can complete on is listed +next to its target and resolves by name. """ import json from collections.abc import Sequence +from itertools import chain +from typing import Final import pytest from fastapi import HTTPException +from pydantic import TypeAdapter from starlette.requests import Request import litellm from litellm import Router from litellm.integrations.custom_logger import CustomLogger +from litellm.integrations.custom_logger_v2 import CallContext, CustomLoggerV2, Field, Hook, Patch, on +from litellm.integrations.custom_logger_v2.enforcement import DeclaredView from litellm.proxy import proxy_server from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.utils import ProxyLogging @@ -38,6 +44,35 @@ class _ListingGate(CustomLogger): return [name for name in model_names if name not in self.hidden] +_LISTED_NAMES: TypeAdapter[tuple[str, ...]] = TypeAdapter(tuple[str, ...]) + + +class _V2ListingGate(CustomLoggerV2): + def __init__(self, hidden: frozenset[str]) -> None: + super().__init__() + self.hidden = hidden + + @on(Hook.list_models, reads={Field.model_list}, writes={Field.model_list}) + async def list_models(self, view: DeclaredView, ctx: CallContext[object]) -> Patch: + names: Final = _LISTED_NAMES.validate_python(view.model_list) + return Patch().set(Field.model_list, tuple(name for name in names if name not in self.hidden)) + + +_LISTING_GATES = pytest.mark.parametrize("gate", [_ListingGate, _V2ListingGate]) + + +class _ListingRecorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.offered: tuple[tuple[str, ...], ...] = () + + async def async_filter_listed_models( + self, user_api_key_dict: UserAPIKeyAuth, model_names: Sequence[str] + ) -> Sequence[str]: + self.offered = (*self.offered, tuple(model_names)) + return model_names + + class _ListingAllowList(CustomLogger): def __init__(self, kept: frozenset[str]) -> None: super().__init__() @@ -306,10 +341,16 @@ async def test_alias_to_a_model_the_listing_hides_is_not_described_by_it( ("model_name_team1_abc", "team-chat", ["gpt-4.1-mini", "fast", "model_name_team1_abc"]), ], ) +@_LISTING_GATES async def test_alias_to_a_model_a_listing_callback_hides_is_not_described_by_it( - router: Router, monkeypatch: pytest.MonkeyPatch, target: str, gated: str, models: list[str] + router: Router, + monkeypatch: pytest.MonkeyPatch, + target: str, + gated: str, + models: list[str], + gate: type[_ListingGate | _V2ListingGate], ) -> None: - monkeypatch.setattr(litellm, "callbacks", [_ListingGate(frozenset({gated}))]) + monkeypatch.setattr(litellm, "callbacks", [gate(frozenset({gated}))]) ProxyLogging._callback_capabilities_cache.clear() caller = _team_member(models=models, team_model_aliases={"fast": target}) @@ -350,3 +391,231 @@ async def test_hidden_router_settings_alias_the_key_does_not_list_is_not_retriev with pytest.raises(HTTPException) as retrieval: await proxy_server.model_info(model_id="quiet", user_api_key_dict=caller) assert retrieval.value.status_code == 404 + + +async def _model_group_rows( + user_api_key_dict: UserAPIKeyAuth, model_group: str | None = None +) -> dict[str, dict[str, object]]: + response: Final = await proxy_server.model_group_info(user_api_key_dict=user_api_key_dict, model_group=model_group) + return {row.model_group: row.model_dump() for row in response["data"]} + + +def _team_listing(*extra: str, **aliases: dict[str, str]) -> UserAPIKeyAuth: + return _team_member(models=["gpt-4.1-mini", "team-chat", *extra], **aliases) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("caller", "alias", "target"), + [ + (_team_listing(aliases={"mini": "gpt-4.1-mini"}), "mini", "gpt-4.1-mini"), + (_team_listing(team_model_aliases={"claude-sonnet-4-5": "gpt-4.1-mini"}), "claude-sonnet-4-5", "gpt-4.1-mini"), + (_team_listing("fast", team_model_aliases={"fast": "gpt-4.1-mini"}), "fast", "gpt-4.1-mini"), + (_team_listing(team_model_aliases={"chat": "team-chat"}), "chat", "team-chat"), + (_alias_holder(aliases={"fast": "gpt-4.1"}), "fast", "gpt-4.1"), + (_alias_holder(router_settings={"model_group_alias": {"fast": "gpt-4.1"}}), "fast", "gpt-4.1"), + ], +) +async def test_model_group_info_describes_every_alias_v1_models_lists_as_its_target( + router: Router, caller: UserAPIKeyAuth, alias: str, target: str +) -> None: + rows: Final = await _model_group_rows(caller) + + assert list(rows) == await _v1_models(caller) + assert rows[alias] == {**rows[target], "model_group": alias} + assert await _model_group_rows(caller, model_group=alias) == {alias: rows[alias]} + + +@pytest.mark.asyncio +async def test_model_group_info_describes_a_team_alias_to_a_model_the_team_does_not_list(router: Router) -> None: + caller: Final = _team_listing("fast", team_model_aliases={"fast": "gpt-4.1"}) + + rows: Final = await _model_group_rows(caller) + assert list(rows) == await _v1_models(caller) + assert rows["fast"] == {**(await _model_group_rows(_keyholder("gpt-4.1")))["gpt-4.1"], "model_group": "fast"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("gate", [None, _ListingGate, _V2ListingGate]) +async def test_model_group_info_describes_a_team_alias_to_a_team_model_the_key_lists_by_internal_name( + router: Router, monkeypatch: pytest.MonkeyPatch, gate: type[_ListingGate | _V2ListingGate] | None +) -> None: + caller: Final = _team_member(team_model_aliases={"chat": "team-chat"}) + monkeypatch.setattr(litellm, "callbacks", [] if gate is None else [gate(frozenset({"unrelated"}))]) + ProxyLogging._callback_capabilities_cache.clear() + + rows: Final = await _model_group_rows(caller) + assert rows["chat"] == {**rows["model_name_team1_abc"], "model_group": "chat"} + assert await _model_group_rows(caller, model_group="chat") == {"chat": rows["chat"]} + + +@pytest.mark.asyncio +@_LISTING_GATES +async def test_a_team_alias_to_a_team_model_a_listing_callback_hides_gets_no_model_group_row( + router: Router, monkeypatch: pytest.MonkeyPatch, gate: type[_ListingGate | _V2ListingGate] +) -> None: + caller: Final = _team_member(team_model_aliases={"chat": "team-chat"}) + monkeypatch.setattr(litellm, "callbacks", [gate(frozenset({"team-chat"}))]) + ProxyLogging._callback_capabilities_cache.clear() + + assert list(await _model_group_rows(caller)) == ["gpt-4.1-mini"] + assert await _model_group_rows(caller, model_group="chat") == {} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("target", "team_id"), [("paused", "team1"), ("hidden", "team1"), ("model_name_team1_abc", "team2")] +) +async def test_model_group_info_does_not_describe_an_alias_to_a_model_the_listing_hides( + router: Router, target: str, team_id: str +) -> None: + models: Final = ["gpt-4.1-mini", "fast"] + caller: Final = _team_member(team_id=team_id, models=models, team_model_aliases={"fast": target}) + + assert await _model_group_rows(caller) == await _model_group_rows(_team_member(team_id=team_id, models=models)) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("models", [["gpt-4.1-mini", "paused"], ["gpt-4.1-mini", "fast", "paused"]]) +async def test_model_group_info_lists_a_paused_model_but_describes_no_alias_by_it( + router: Router, models: list[str] +) -> None: + caller: Final = _team_member(models=models, team_model_aliases={"fast": "paused"}) + + assert await _model_group_rows(caller) == await _model_group_rows(_team_member(models=models)) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("target", "gated", "models"), + [ + ("gpt-4.1", "gpt-4.1", ["gpt-4.1-mini", "fast"]), + ("gpt-4.1", "gpt-4.1", ["gpt-4.1-mini", "fast", "gpt-4.1"]), + ("model_name_team1_abc", "team-chat", ["gpt-4.1-mini", "fast", "model_name_team1_abc"]), + ], +) +@_LISTING_GATES +async def test_model_group_info_hides_an_alias_whose_target_a_listing_callback_hides( + router: Router, + monkeypatch: pytest.MonkeyPatch, + target: str, + gated: str, + models: list[str], + gate: type[_ListingGate | _V2ListingGate], +) -> None: + monkeypatch.setattr(litellm, "callbacks", [gate(frozenset({gated}))]) + ProxyLogging._callback_capabilities_cache.clear() + caller: Final = _team_member(models=models, team_model_aliases={"fast": target}) + + assert await _model_group_rows(caller) == await _model_group_rows(_team_member(models=models)) + + +@pytest.mark.asyncio +async def test_model_group_info_keeps_every_group_an_allow_list_callback_allows_when_an_alias_points_past_them( + router: Router, monkeypatch: pytest.MonkeyPatch +) -> None: + caller: Final = _team_member(models=["gpt-4.1-mini", "legacy", "fast"], team_model_aliases={"fast": "gpt-4.1"}) + listed: Final = await _v1_models(caller) + + monkeypatch.setattr(litellm, "callbacks", [_ListingAllowList(frozenset(listed))]) + ProxyLogging._callback_capabilities_cache.clear() + assert list(await _model_group_rows(caller)) == listed + + +@pytest.mark.asyncio +@_LISTING_GATES +async def test_a_name_a_listing_callback_hides_stays_hidden_when_it_is_also_an_alias( + router: Router, monkeypatch: pytest.MonkeyPatch, gate: type[_ListingGate | _V2ListingGate] +) -> None: + monkeypatch.setattr(litellm, "callbacks", [gate(frozenset({"fast"}))]) + ProxyLogging._callback_capabilities_cache.clear() + models: Final = ["gpt-4.1-mini", "fast"] + caller: Final = _team_member(models=models, team_model_aliases={"fast": "gpt-4.1-mini"}) + unaliased: Final = _team_member(models=models) + + assert await _v1_models(caller) == await _v1_models(unaliased) + assert await _model_group_rows(caller) == await _model_group_rows(unaliased) + with pytest.raises(HTTPException) as retrieval: + await proxy_server.model_info(model_id="fast", user_api_key_dict=caller) + with pytest.raises(HTTPException) as unaliased_retrieval: + await proxy_server.model_info(model_id="fast", user_api_key_dict=unaliased) + assert retrieval.value.status_code == unaliased_retrieval.value.status_code + + +@pytest.mark.asyncio +@_LISTING_GATES +async def test_an_alias_to_an_unlisted_model_a_listing_callback_keeps_is_described_by_that_model( + router: Router, monkeypatch: pytest.MonkeyPatch, gate: type[_ListingGate | _V2ListingGate] +) -> None: + caller: Final = _team_listing("fast", team_model_aliases={"fast": "gpt-4.1"}) + target_holder: Final = _keyholder("gpt-4.1") + target_row: Final = {row["id"]: row for row in (await proxy_server.model_list(user_api_key_dict=target_holder, request=None))["data"]} + target_info: Final = await proxy_server.model_info(model_id="gpt-4.1", user_api_key_dict=target_holder) + target_group: Final = {**(await _model_group_rows(target_holder))["gpt-4.1"], "model_group": "fast"} + monkeypatch.setattr(litellm, "callbacks", [gate(frozenset({"unrelated"}))]) + ProxyLogging._callback_capabilities_cache.clear() + + rows: Final = {row["id"]: row for row in (await proxy_server.model_list(user_api_key_dict=caller, request=None))["data"]} + assert rows["fast"] == {**target_row["gpt-4.1"], "id": "fast"} + assert await proxy_server.model_info(model_id="fast", user_api_key_dict=caller) == {**target_info, "id": "fast"} + assert (await _model_group_rows(caller))["fast"] == target_group + assert await _model_group_rows(caller, model_group="fast") == {"fast": target_group} + + +@pytest.mark.asyncio +@_LISTING_GATES +async def test_an_alias_to_an_unlisted_model_a_listing_callback_hides_is_not_described_by_that_model( + router: Router, monkeypatch: pytest.MonkeyPatch, gate: type[_ListingGate | _V2ListingGate] +) -> None: + caller: Final = _team_listing("fast", team_model_aliases={"fast": "gpt-4.1"}) + monkeypatch.setattr(litellm, "callbacks", [gate(frozenset({"gpt-4.1"}))]) + ProxyLogging._callback_capabilities_cache.clear() + + assert (await _model_group_rows(caller))["fast"]["providers"] == [] + with pytest.raises(HTTPException) as lookup: + await proxy_server.model_info(model_id="fast", user_api_key_dict=caller) + assert lookup.value.status_code == 404 + + +@pytest.mark.asyncio +@_LISTING_GATES +async def test_an_alias_named_like_a_router_alias_to_a_hidden_model_is_listed_as_its_own_target( + router: Router, monkeypatch: pytest.MonkeyPatch, gate: type[_ListingGate | _V2ListingGate] +) -> None: + monkeypatch.setattr(litellm, "callbacks", [gate(frozenset({"gpt-4.1"}))]) + ProxyLogging._callback_capabilities_cache.clear() + caller: Final = UserAPIKeyAuth( + api_key="sk-test", + user_id="u", + user_role=LitellmUserRoles.INTERNAL_USER, + models=["gpt-4.1-mini", "gpt-4.1"], + aliases={"legacy": "gpt-4.1-mini"}, + ) + + listing: Final = await proxy_server.model_list(user_api_key_dict=caller, request=None) + rows: Final = {row["id"]: row for row in listing["data"]} + assert rows["legacy"] == {**rows["gpt-4.1-mini"], "id": "legacy"} + target: Final = await proxy_server.model_info(model_id="gpt-4.1-mini", user_api_key_dict=caller) + assert await proxy_server.model_info(model_id="legacy", user_api_key_dict=caller) == {**target, "id": "legacy"} + groups: Final = await _model_group_rows(caller) + assert groups["legacy"] == {**groups["gpt-4.1-mini"], "model_group": "legacy"} + assert await _model_group_rows(caller, model_group="legacy") == {"legacy": groups["legacy"]} + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("model_group", "offered"), [("gpt-4.1-mini", {"gpt-4.1-mini"}), ("fast", {"gpt-4.1"})]) +async def test_model_group_info_for_one_group_offers_listing_callbacks_only_the_names_it_reads( + router: Router, monkeypatch: pytest.MonkeyPatch, model_group: str, offered: set[str] +) -> None: + recorder: Final = _ListingRecorder() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + ProxyLogging._callback_capabilities_cache.clear() + caller: Final = _team_member( + models=["gpt-4.1-mini", "gpt-4.1", "model_name_team1_abc"], team_model_aliases={"fast": "gpt-4.1"} + ) + + rows: Final = await _model_group_rows(caller, model_group=model_group) + + assert list(rows) == [model_group] + assert set(chain.from_iterable(recorder.offered)) == offered + assert rows[model_group] == (await _model_group_rows(caller))[model_group]