mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-14 09:03:16 +00:00
fix(proxy): list key and team aliases on /model_group/info and apply listing callbacks to them (#51754)
* fix(proxy): list key and team model aliases on /model_group/info
* fix(proxy): keep listing-callback and pause filters on model alias rows
A name a legacy or V2 listing callback hides stays hidden on /v1/models,
/v1/models/{model_id} and /model_group/info even when it is also an alias.
V2 Hook.list_models subscribers now limit which target an alias row
describes, /model_group/info describes no alias by a paused model, and
?model_group=<name> offers listing callbacks only the names that row reads.
* test(proxy): annotate the new alias test locals as Final
* fix(proxy): keep a caller alias listed when a router alias of the same name points at a hidden model
A key or team alias always wins over a router model_group_alias of the same name, so alias rows are now dropped only for names a listing callback hid itself, not for router aliases whose target it hid
* fix(proxy): describe an alias by an unlisted target only when listing callbacks keep it
Listing callbacks are now also offered the unlisted model each listed key or team alias
sends requests to, and the model a router alias of that name resolves to. The alias row
is described by that model only when the callback keeps every one of those names, and a
name the callback hides there hides no listed name. Before, any installed listing
callback, V2 included, left such an alias with no limits and a 404 lookup
* fix(proxy): describe an alias to a team model's public name when the key lists its internal name
/model_group/info lists team models by their internal routing name, so an alias to the
public name found no listed target and got no row while /v1/models listed it. Alias
resolution now reads the listed names by their public names. The listing callback
docstring also names litellm.model_alias_map among the alias sources.
---------
Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
This commit is contained in:
tavren27
GitHub
parent
8973e2dc4f
commit
d76cdcbb8a
5 changed files with 487 additions and 50 deletions
|
|
@ -440,7 +440,12 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
|
|||
"""Runs on the model listing routes (`/v1/models`, `/v1/models/{id}`, `/model/info`,
|
||||
`/model_group/info`) with the public model names the route would otherwise return, so a
|
||||
lookup of one model may offer just that name: decide per name, never by position in the
|
||||
sequence. Return the names to keep as a sequence of strings; a name left out disappears
|
||||
sequence. The names also include each unlisted model that a key alias, a team alias, or
|
||||
`litellm.model_alias_map` sends a listed name to without checking the caller's access to that
|
||||
model, and the model a `model_group_alias` of that name resolves to, and the listed name's row
|
||||
carries that model's info only while the callback keeps all of them. Leaving one out hides
|
||||
no listed name. Alias names a key or team maps without listing them are not offered. Return
|
||||
the names to keep as a sequence of strings; a name left out disappears
|
||||
from every listing, any alias of it offered in the same call goes with it, and
|
||||
`/v1/models/{id}` answers 404 for it, exactly as for a model that does not exist. Names
|
||||
outside `model_names` are ignored, so a callback can only narrow the listing, never widen
|
||||
|
|
|
|||
|
|
@ -352,6 +352,18 @@ def alias_listing_entries(
|
|||
return (*retargeted, *added)
|
||||
|
||||
|
||||
def unlisted_alias_targets(entries: Sequence[tuple[str, str]], aliases: CallerAliases) -> tuple[str, ...]:
|
||||
layers: Final = _alias_layers(aliases)
|
||||
listed: Final = frozenset(chain.from_iterable(entries))
|
||||
return tuple(
|
||||
dict.fromkeys(
|
||||
target
|
||||
for target, checks_target in (_dispatch(response_id, layers) for response_id, _ in entries)
|
||||
if not checks_target and target not in listed
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def label_entries(listed: Sequence[tuple[str, str]], entries: Sequence[tuple[str, str]]) -> tuple[tuple[str, str], ...]:
|
||||
"""`entries` with every `listed` name read under its own lookup id again, so an alias
|
||||
`alias_listing_entries` describes by its target keeps its own display name, the way a
|
||||
|
|
@ -448,7 +460,7 @@ class TeamModelNameTranslator:
|
|||
@staticmethod
|
||||
def _response_to_lookup_map(
|
||||
model_names: Sequence[str],
|
||||
internal_to_public: dict[str, str],
|
||||
internal_to_public: Mapping[str, str],
|
||||
) -> dict[str, str]:
|
||||
"""Map each public response id to the first internal lookup id seen in
|
||||
`model_names`, preserving first-occurrence order. First-wins keeps list
|
||||
|
|
@ -484,6 +496,12 @@ class TeamModelNameTranslator:
|
|||
return [(name, name) for name in model_names]
|
||||
return list(TeamModelNameTranslator._response_to_lookup_map(model_names, internal_to_public).items())
|
||||
|
||||
@staticmethod
|
||||
def public_name_entries(
|
||||
model_names: Sequence[str], internal_to_public: Mapping[str, str]
|
||||
) -> tuple[tuple[str, str], ...]:
|
||||
return tuple(TeamModelNameTranslator._response_to_lookup_map(model_names, internal_to_public).items())
|
||||
|
||||
@staticmethod
|
||||
def translate_listing(
|
||||
model_names: list[str],
|
||||
|
|
|
|||
|
|
@ -582,7 +582,9 @@ from litellm.proxy.common_utils.model_listing_utils import (
|
|||
caller_alias_maps,
|
||||
claude_code_view_ids,
|
||||
configured_display_names,
|
||||
dispatched_model,
|
||||
label_entries,
|
||||
unlisted_alias_targets,
|
||||
)
|
||||
from litellm.proxy.common_utils.openai_endpoint_utils import (
|
||||
remove_sensitive_info_from_deployment,
|
||||
|
|
@ -12768,7 +12770,10 @@ class ProxyStartupEvent:
|
|||
async def _names_hidden_by_listing_callbacks(
|
||||
user_api_key_dict: UserAPIKeyAuth, model_names: Sequence[str]
|
||||
) -> frozenset[str]:
|
||||
hidden: Final = await proxy_logging_obj.hidden_by_listing_callbacks(user_api_key_dict, model_names)
|
||||
return _with_router_aliases_to(await proxy_logging_obj.hidden_by_listing_callbacks(user_api_key_dict, model_names))
|
||||
|
||||
|
||||
def _with_router_aliases_to(hidden: frozenset[str]) -> frozenset[str]:
|
||||
if not hidden or llm_router is None:
|
||||
return hidden
|
||||
aliases: Final = llm_router.registry().aliases()
|
||||
|
|
@ -12781,11 +12786,47 @@ async def _names_hidden_by_listing_callbacks(
|
|||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _ListingCallbackVerdict:
|
||||
hid: frozenset[str]
|
||||
hidden: frozenset[str]
|
||||
kept_alias_targets: frozenset[str]
|
||||
|
||||
|
||||
async def _listing_callback_verdict(
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
listed_names: Sequence[str],
|
||||
alias_targets: Sequence[str],
|
||||
internal_to_public: Mapping[str, str],
|
||||
) -> _ListingCallbackVerdict:
|
||||
router_aliases: Final = llm_router.registry().aliases() if llm_router is not None else {}
|
||||
target_reads: Final = MappingProxyType(
|
||||
{
|
||||
target: frozenset(
|
||||
internal_to_public.get(name, name)
|
||||
for name in (target, resolve_model_group_alias(router_aliases, target))
|
||||
if name is not None
|
||||
)
|
||||
for target in alias_targets
|
||||
}
|
||||
)
|
||||
offered_hid: Final = await proxy_logging_obj.hidden_by_listing_callbacks(
|
||||
user_api_key_dict, tuple(dict.fromkeys((*listed_names, *chain.from_iterable(target_reads.values()))))
|
||||
)
|
||||
hid: Final = offered_hid.intersection(listed_names)
|
||||
return _ListingCallbackVerdict(
|
||||
hid,
|
||||
_with_router_aliases_to(hid),
|
||||
frozenset(target for target, names in target_reads.items() if offered_hid.isdisjoint(names)),
|
||||
)
|
||||
|
||||
|
||||
def _describable_model_groups(
|
||||
listed_entries: Sequence[tuple[str, str]],
|
||||
hidden_names: frozenset[str] | set[str],
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
team_id: str | None,
|
||||
kept_alias_targets: frozenset[str],
|
||||
) -> DeployedModelGroups:
|
||||
listing_team_id: Final = team_id or user_api_key_dict.team_id
|
||||
return DeployedModelGroups(
|
||||
|
|
@ -12793,8 +12834,8 @@ def _describable_model_groups(
|
|||
hidden_names | undiscoverable_model_groups(llm_router, user_api_key_dict, listing_team_id),
|
||||
listing_team_id,
|
||||
(
|
||||
frozenset(lookup_id for _, lookup_id in listed_entries)
|
||||
if ProxyLogging.callback_capabilities().listed_models_filters
|
||||
frozenset((*(lookup_id for _, lookup_id in listed_entries), *kept_alias_targets))
|
||||
if ProxyLogging.filters_listed_models()
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
|
@ -12896,16 +12937,26 @@ async def _key_model_listing(
|
|||
excluded: Final = hidden_names | undiscoverable_model_names(
|
||||
names, llm_router, user_api_key_dict, team_id or user_api_key_dict.team_id
|
||||
)
|
||||
listed_entries: Final = await _entries_kept_by_listing_callbacks(
|
||||
TeamModelNameTranslator.listing_entries(
|
||||
[name for name in names if name not in excluded], llm_router, general_settings_view()
|
||||
),
|
||||
user_api_key_dict,
|
||||
translated_entries: Final = TeamModelNameTranslator.listing_entries(
|
||||
[name for name in names if name not in excluded], llm_router, general_settings_view()
|
||||
)
|
||||
entries: Final = alias_listing_entries(
|
||||
listed_entries,
|
||||
caller_aliases,
|
||||
_describable_model_groups(listed_entries, hidden_names, user_api_key_dict, team_id),
|
||||
verdict: Final = await _listing_callback_verdict(
|
||||
user_api_key_dict,
|
||||
tuple(response_id for response_id, _ in translated_entries),
|
||||
unlisted_alias_targets(translated_entries, caller_aliases),
|
||||
TeamModelNameTranslator.build_internal_to_public_map(llm_router, general_settings_view()),
|
||||
)
|
||||
listed_entries: Final = tuple(entry for entry in translated_entries if entry[0] not in verdict.hidden)
|
||||
entries: Final = tuple(
|
||||
entry
|
||||
for entry in alias_listing_entries(
|
||||
listed_entries,
|
||||
caller_aliases,
|
||||
_describable_model_groups(
|
||||
listed_entries, hidden_names, user_api_key_dict, team_id, verdict.kept_alias_targets
|
||||
),
|
||||
)
|
||||
if entry[0] not in verdict.hid
|
||||
)
|
||||
return _KeyModelListing(
|
||||
rows=tuple(
|
||||
|
|
@ -13248,18 +13299,18 @@ async def model_info(
|
|||
hidden_names: Final = blocked_names | unhealthy_names
|
||||
internal_to_public: Final = TeamModelNameTranslator.build_internal_to_public_map(llm_router, settings)
|
||||
caller_aliases: Final = await request_caller_aliases(user_api_key_dict, team_id)
|
||||
callback_hidden_names: Final = await _names_hidden_by_listing_callbacks(
|
||||
user_api_key_dict,
|
||||
tuple(
|
||||
response_id
|
||||
for response_id, _ in TeamModelNameTranslator.listing_entries(
|
||||
tuple(m for m in all_models if m not in hidden_names), llm_router, settings
|
||||
)
|
||||
),
|
||||
offered_entries: Final = TeamModelNameTranslator.listing_entries(
|
||||
tuple(m for m in all_models if m not in hidden_names), llm_router, settings
|
||||
)
|
||||
if hidden_names or callback_hidden_names:
|
||||
verdict: Final = await _listing_callback_verdict(
|
||||
user_api_key_dict,
|
||||
tuple(response_id for response_id, _ in offered_entries),
|
||||
unlisted_alias_targets(offered_entries, caller_aliases),
|
||||
internal_to_public,
|
||||
)
|
||||
if hidden_names or verdict.hidden:
|
||||
all_models = [
|
||||
m for m in all_models if m not in hidden_names and internal_to_public.get(m, m) not in callback_hidden_names
|
||||
m for m in all_models if m not in hidden_names and internal_to_public.get(m, m) not in verdict.hidden
|
||||
]
|
||||
undiscoverable_names: Final = undiscoverable_model_names(
|
||||
all_models, llm_router, user_api_key_dict, team_id or user_api_key_dict.team_id
|
||||
|
|
@ -13268,11 +13319,17 @@ async def model_info(
|
|||
listed_entries: Final = TeamModelNameTranslator.listing_entries(
|
||||
tuple(m for m in all_models if m not in undiscoverable_names), llm_router, settings
|
||||
)
|
||||
aliased_model_id: Final = alias_target(
|
||||
model_id,
|
||||
caller_aliases,
|
||||
listed_entries,
|
||||
_describable_model_groups(listed_entries, hidden_names, user_api_key_dict, team_id),
|
||||
aliased_model_id: Final = (
|
||||
None
|
||||
if model_id in verdict.hid
|
||||
else alias_target(
|
||||
model_id,
|
||||
caller_aliases,
|
||||
listed_entries,
|
||||
_describable_model_groups(
|
||||
listed_entries, hidden_names, user_api_key_dict, team_id, verdict.kept_alias_targets
|
||||
),
|
||||
)
|
||||
)
|
||||
resolved_model_id: Final = TeamModelNameTranslator.resolve_public_name(
|
||||
model_id=aliased_model_id or model_id,
|
||||
|
|
@ -19314,39 +19371,123 @@ async def model_group_info(
|
|||
undiscoverable_group_names: Final = undiscoverable_model_names(
|
||||
all_models_str, llm_router, user_api_key_dict, user_api_key_dict.team_id
|
||||
)
|
||||
listed_group_names: Final = tuple(name for name in all_models_str if name not in undiscoverable_group_names)
|
||||
caller_aliases: Final = await request_caller_aliases(user_api_key_dict)
|
||||
requested_names: Final = (
|
||||
None
|
||||
if model_group is None
|
||||
else _names_a_model_group_row_reads(model_group, caller_aliases, llm_router.registry().aliases())
|
||||
)
|
||||
internal_to_public: Final = TeamModelNameTranslator.build_internal_to_public_map(llm_router, general_settings)
|
||||
listed_group_names: Final = tuple(
|
||||
dict.fromkeys(
|
||||
name
|
||||
for name in all_models_str
|
||||
if name not in undiscoverable_group_names
|
||||
and (requested_names is None or not requested_names.isdisjoint((name, internal_to_public.get(name, name))))
|
||||
)
|
||||
)
|
||||
|
||||
# Append A2A agents to model groups
|
||||
from litellm.proxy.agent_endpoints.model_list_helpers import (
|
||||
append_agents_to_model_group,
|
||||
)
|
||||
|
||||
model_groups: Final = await append_agents_to_model_group(
|
||||
model_groups=get_model_group_info(
|
||||
llm_router=llm_router,
|
||||
all_models_str=listed_group_names,
|
||||
model_group=model_group,
|
||||
team_id=user_api_key_dict.team_id,
|
||||
agent_groups: Final = await append_agents_to_model_group(model_groups=[], user_api_key_dict=user_api_key_dict)
|
||||
verdict: Final = await _listing_callback_verdict(
|
||||
user_api_key_dict,
|
||||
tuple(
|
||||
dict.fromkeys(
|
||||
(
|
||||
*(internal_to_public.get(name, name) for name in listed_group_names),
|
||||
*(group.model_group for group in agent_groups),
|
||||
)
|
||||
)
|
||||
),
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
unlisted_alias_targets(
|
||||
TeamModelNameTranslator.public_name_entries(listed_group_names, internal_to_public), caller_aliases
|
||||
),
|
||||
internal_to_public,
|
||||
)
|
||||
internal_to_public: Final = TeamModelNameTranslator.build_internal_to_public_map(llm_router, general_settings)
|
||||
public_group_names: Final = tuple(
|
||||
internal_to_public.get(group.model_group, group.model_group) for group in model_groups
|
||||
kept_entries: Final = tuple(
|
||||
(name, name) for name in listed_group_names if internal_to_public.get(name, name) not in verdict.hidden
|
||||
)
|
||||
callback_hidden_names: Final = await _names_hidden_by_listing_callbacks(
|
||||
user_api_key_dict, tuple(dict.fromkeys(public_group_names))
|
||||
blocked_names: Final = llm_router.health().fully_blocked_model_names()
|
||||
requested_entries: Final = tuple(
|
||||
entry
|
||||
for entry in _listing_with_aliases_to_unpaused_models(
|
||||
kept_entries,
|
||||
TeamModelNameTranslator.public_name_entries(
|
||||
tuple(name for name, _ in kept_entries if name not in blocked_names), internal_to_public
|
||||
),
|
||||
caller_aliases,
|
||||
_describable_model_groups(kept_entries, blocked_names, user_api_key_dict, None, verdict.kept_alias_targets),
|
||||
)
|
||||
if (model_group is None or entry[0] == model_group) and entry[0] not in verdict.hid
|
||||
)
|
||||
lookup_ids: Final = tuple(dict.fromkeys(lookup_id for _, lookup_id in requested_entries))
|
||||
groups_by_lookup_id: Final = MappingProxyType(
|
||||
dict(
|
||||
zip(
|
||||
lookup_ids,
|
||||
get_model_group_info(
|
||||
llm_router=llm_router,
|
||||
all_models_str=lookup_ids,
|
||||
model_group=None,
|
||||
team_id=user_api_key_dict.team_id,
|
||||
),
|
||||
strict=True,
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
return {
|
||||
"data": [
|
||||
group
|
||||
for group, public_name in zip(model_groups, public_group_names, strict=True)
|
||||
if public_name not in callback_hidden_names
|
||||
*(
|
||||
_model_group_row_listed_as(groups_by_lookup_id[lookup_id], listed_name, lookup_id)
|
||||
for listed_name, lookup_id in requested_entries
|
||||
),
|
||||
*(group for group in agent_groups if group.model_group not in verdict.hidden),
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
def _names_a_model_group_row_reads(model_group: str, aliases: CallerAliases, router_aliases: object) -> frozenset[str]:
|
||||
return frozenset(
|
||||
name
|
||||
for name in (
|
||||
model_group,
|
||||
dispatched_model(model_group, aliases),
|
||||
resolve_model_group_alias(router_aliases, model_group),
|
||||
)
|
||||
if name is not None
|
||||
)
|
||||
|
||||
|
||||
def _listing_with_aliases_to_unpaused_models(
|
||||
listed_entries: Sequence[tuple[str, str]],
|
||||
unpaused_public_entries: Sequence[tuple[str, str]],
|
||||
aliases: CallerAliases,
|
||||
deployed: Container[str],
|
||||
) -> tuple[tuple[str, str], ...]:
|
||||
described: Final = alias_listing_entries(unpaused_public_entries, aliases, deployed)
|
||||
lookup_by_name: Final = MappingProxyType(dict(described))
|
||||
shown_names: Final = frozenset(name for name, _ in (*listed_entries, *unpaused_public_entries))
|
||||
return (
|
||||
*((name, lookup_by_name.get(name, lookup_id)) for name, lookup_id in listed_entries),
|
||||
*(entry for entry in described if entry[0] not in shown_names),
|
||||
)
|
||||
|
||||
|
||||
def _model_group_row_listed_as(row: ModelGroupInfoProxy, listed_name: str, lookup_id: str) -> ModelGroupInfoProxy:
|
||||
if listed_name == lookup_id:
|
||||
return row
|
||||
return row.model_copy(
|
||||
update={
|
||||
"model_group": listed_name,
|
||||
"is_public_model_group": listed_name in (litellm.public_model_groups or ()),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/model/settings",
|
||||
description="Returns provider name, description, and required parameters for each provider",
|
||||
|
|
|
|||
|
|
@ -4977,6 +4977,10 @@ class ProxyLogging:
|
|||
if result is not None:
|
||||
merged_headers.update(result)
|
||||
|
||||
@staticmethod
|
||||
def filters_listed_models() -> bool:
|
||||
return bool(hook_subscribers(Hook.list_models, ProxyLogging.callback_capabilities().hook_callbacks).entries)
|
||||
|
||||
async def hidden_by_listing_callbacks(
|
||||
self, user_api_key_dict: UserAPIKeyAuth, model_names: Sequence[str]
|
||||
) -> frozenset[str]:
|
||||
|
|
|
|||
|
|
@ -1,19 +1,25 @@
|
|||
"""
|
||||
Tests for key and team `model_aliases` on the model listing endpoints: GET /v1/models
|
||||
(`model_list`, OpenAI and Anthropic shapes) and GET /v1/models/{id} (`model_info`).
|
||||
An alias the caller can complete on is listed next to its target and resolves by name.
|
||||
(`model_list`, OpenAI and Anthropic shapes), GET /v1/models/{id} (`model_info`) and
|
||||
GET /model_group/info (`model_group_info`). An alias the caller can complete on is listed
|
||||
next to its target and resolves by name.
|
||||
"""
|
||||
|
||||
import json
|
||||
from collections.abc import Sequence
|
||||
from itertools import chain
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
from pydantic import TypeAdapter
|
||||
from starlette.requests import Request
|
||||
|
||||
import litellm
|
||||
from litellm import Router
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.integrations.custom_logger_v2 import CallContext, CustomLoggerV2, Field, Hook, Patch, on
|
||||
from litellm.integrations.custom_logger_v2.enforcement import DeclaredView
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
|
||||
from litellm.proxy.utils import ProxyLogging
|
||||
|
|
@ -38,6 +44,35 @@ class _ListingGate(CustomLogger):
|
|||
return [name for name in model_names if name not in self.hidden]
|
||||
|
||||
|
||||
_LISTED_NAMES: TypeAdapter[tuple[str, ...]] = TypeAdapter(tuple[str, ...])
|
||||
|
||||
|
||||
class _V2ListingGate(CustomLoggerV2):
|
||||
def __init__(self, hidden: frozenset[str]) -> None:
|
||||
super().__init__()
|
||||
self.hidden = hidden
|
||||
|
||||
@on(Hook.list_models, reads={Field.model_list}, writes={Field.model_list})
|
||||
async def list_models(self, view: DeclaredView, ctx: CallContext[object]) -> Patch:
|
||||
names: Final = _LISTED_NAMES.validate_python(view.model_list)
|
||||
return Patch().set(Field.model_list, tuple(name for name in names if name not in self.hidden))
|
||||
|
||||
|
||||
_LISTING_GATES = pytest.mark.parametrize("gate", [_ListingGate, _V2ListingGate])
|
||||
|
||||
|
||||
class _ListingRecorder(CustomLogger):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.offered: tuple[tuple[str, ...], ...] = ()
|
||||
|
||||
async def async_filter_listed_models(
|
||||
self, user_api_key_dict: UserAPIKeyAuth, model_names: Sequence[str]
|
||||
) -> Sequence[str]:
|
||||
self.offered = (*self.offered, tuple(model_names))
|
||||
return model_names
|
||||
|
||||
|
||||
class _ListingAllowList(CustomLogger):
|
||||
def __init__(self, kept: frozenset[str]) -> None:
|
||||
super().__init__()
|
||||
|
|
@ -306,10 +341,16 @@ async def test_alias_to_a_model_the_listing_hides_is_not_described_by_it(
|
|||
("model_name_team1_abc", "team-chat", ["gpt-4.1-mini", "fast", "model_name_team1_abc"]),
|
||||
],
|
||||
)
|
||||
@_LISTING_GATES
|
||||
async def test_alias_to_a_model_a_listing_callback_hides_is_not_described_by_it(
|
||||
router: Router, monkeypatch: pytest.MonkeyPatch, target: str, gated: str, models: list[str]
|
||||
router: Router,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
target: str,
|
||||
gated: str,
|
||||
models: list[str],
|
||||
gate: type[_ListingGate | _V2ListingGate],
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "callbacks", [_ListingGate(frozenset({gated}))])
|
||||
monkeypatch.setattr(litellm, "callbacks", [gate(frozenset({gated}))])
|
||||
ProxyLogging._callback_capabilities_cache.clear()
|
||||
caller = _team_member(models=models, team_model_aliases={"fast": target})
|
||||
|
||||
|
|
@ -350,3 +391,231 @@ async def test_hidden_router_settings_alias_the_key_does_not_list_is_not_retriev
|
|||
with pytest.raises(HTTPException) as retrieval:
|
||||
await proxy_server.model_info(model_id="quiet", user_api_key_dict=caller)
|
||||
assert retrieval.value.status_code == 404
|
||||
|
||||
|
||||
async def _model_group_rows(
|
||||
user_api_key_dict: UserAPIKeyAuth, model_group: str | None = None
|
||||
) -> dict[str, dict[str, object]]:
|
||||
response: Final = await proxy_server.model_group_info(user_api_key_dict=user_api_key_dict, model_group=model_group)
|
||||
return {row.model_group: row.model_dump() for row in response["data"]}
|
||||
|
||||
|
||||
def _team_listing(*extra: str, **aliases: dict[str, str]) -> UserAPIKeyAuth:
|
||||
return _team_member(models=["gpt-4.1-mini", "team-chat", *extra], **aliases)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("caller", "alias", "target"),
|
||||
[
|
||||
(_team_listing(aliases={"mini": "gpt-4.1-mini"}), "mini", "gpt-4.1-mini"),
|
||||
(_team_listing(team_model_aliases={"claude-sonnet-4-5": "gpt-4.1-mini"}), "claude-sonnet-4-5", "gpt-4.1-mini"),
|
||||
(_team_listing("fast", team_model_aliases={"fast": "gpt-4.1-mini"}), "fast", "gpt-4.1-mini"),
|
||||
(_team_listing(team_model_aliases={"chat": "team-chat"}), "chat", "team-chat"),
|
||||
(_alias_holder(aliases={"fast": "gpt-4.1"}), "fast", "gpt-4.1"),
|
||||
(_alias_holder(router_settings={"model_group_alias": {"fast": "gpt-4.1"}}), "fast", "gpt-4.1"),
|
||||
],
|
||||
)
|
||||
async def test_model_group_info_describes_every_alias_v1_models_lists_as_its_target(
|
||||
router: Router, caller: UserAPIKeyAuth, alias: str, target: str
|
||||
) -> None:
|
||||
rows: Final = await _model_group_rows(caller)
|
||||
|
||||
assert list(rows) == await _v1_models(caller)
|
||||
assert rows[alias] == {**rows[target], "model_group": alias}
|
||||
assert await _model_group_rows(caller, model_group=alias) == {alias: rows[alias]}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_model_group_info_describes_a_team_alias_to_a_model_the_team_does_not_list(router: Router) -> None:
|
||||
caller: Final = _team_listing("fast", team_model_aliases={"fast": "gpt-4.1"})
|
||||
|
||||
rows: Final = await _model_group_rows(caller)
|
||||
assert list(rows) == await _v1_models(caller)
|
||||
assert rows["fast"] == {**(await _model_group_rows(_keyholder("gpt-4.1")))["gpt-4.1"], "model_group": "fast"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("gate", [None, _ListingGate, _V2ListingGate])
|
||||
async def test_model_group_info_describes_a_team_alias_to_a_team_model_the_key_lists_by_internal_name(
|
||||
router: Router, monkeypatch: pytest.MonkeyPatch, gate: type[_ListingGate | _V2ListingGate] | None
|
||||
) -> None:
|
||||
caller: Final = _team_member(team_model_aliases={"chat": "team-chat"})
|
||||
monkeypatch.setattr(litellm, "callbacks", [] if gate is None else [gate(frozenset({"unrelated"}))])
|
||||
ProxyLogging._callback_capabilities_cache.clear()
|
||||
|
||||
rows: Final = await _model_group_rows(caller)
|
||||
assert rows["chat"] == {**rows["model_name_team1_abc"], "model_group": "chat"}
|
||||
assert await _model_group_rows(caller, model_group="chat") == {"chat": rows["chat"]}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@_LISTING_GATES
|
||||
async def test_a_team_alias_to_a_team_model_a_listing_callback_hides_gets_no_model_group_row(
|
||||
router: Router, monkeypatch: pytest.MonkeyPatch, gate: type[_ListingGate | _V2ListingGate]
|
||||
) -> None:
|
||||
caller: Final = _team_member(team_model_aliases={"chat": "team-chat"})
|
||||
monkeypatch.setattr(litellm, "callbacks", [gate(frozenset({"team-chat"}))])
|
||||
ProxyLogging._callback_capabilities_cache.clear()
|
||||
|
||||
assert list(await _model_group_rows(caller)) == ["gpt-4.1-mini"]
|
||||
assert await _model_group_rows(caller, model_group="chat") == {}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("target", "team_id"), [("paused", "team1"), ("hidden", "team1"), ("model_name_team1_abc", "team2")]
|
||||
)
|
||||
async def test_model_group_info_does_not_describe_an_alias_to_a_model_the_listing_hides(
|
||||
router: Router, target: str, team_id: str
|
||||
) -> None:
|
||||
models: Final = ["gpt-4.1-mini", "fast"]
|
||||
caller: Final = _team_member(team_id=team_id, models=models, team_model_aliases={"fast": target})
|
||||
|
||||
assert await _model_group_rows(caller) == await _model_group_rows(_team_member(team_id=team_id, models=models))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("models", [["gpt-4.1-mini", "paused"], ["gpt-4.1-mini", "fast", "paused"]])
|
||||
async def test_model_group_info_lists_a_paused_model_but_describes_no_alias_by_it(
|
||||
router: Router, models: list[str]
|
||||
) -> None:
|
||||
caller: Final = _team_member(models=models, team_model_aliases={"fast": "paused"})
|
||||
|
||||
assert await _model_group_rows(caller) == await _model_group_rows(_team_member(models=models))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("target", "gated", "models"),
|
||||
[
|
||||
("gpt-4.1", "gpt-4.1", ["gpt-4.1-mini", "fast"]),
|
||||
("gpt-4.1", "gpt-4.1", ["gpt-4.1-mini", "fast", "gpt-4.1"]),
|
||||
("model_name_team1_abc", "team-chat", ["gpt-4.1-mini", "fast", "model_name_team1_abc"]),
|
||||
],
|
||||
)
|
||||
@_LISTING_GATES
|
||||
async def test_model_group_info_hides_an_alias_whose_target_a_listing_callback_hides(
|
||||
router: Router,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
target: str,
|
||||
gated: str,
|
||||
models: list[str],
|
||||
gate: type[_ListingGate | _V2ListingGate],
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "callbacks", [gate(frozenset({gated}))])
|
||||
ProxyLogging._callback_capabilities_cache.clear()
|
||||
caller: Final = _team_member(models=models, team_model_aliases={"fast": target})
|
||||
|
||||
assert await _model_group_rows(caller) == await _model_group_rows(_team_member(models=models))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_model_group_info_keeps_every_group_an_allow_list_callback_allows_when_an_alias_points_past_them(
|
||||
router: Router, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
caller: Final = _team_member(models=["gpt-4.1-mini", "legacy", "fast"], team_model_aliases={"fast": "gpt-4.1"})
|
||||
listed: Final = await _v1_models(caller)
|
||||
|
||||
monkeypatch.setattr(litellm, "callbacks", [_ListingAllowList(frozenset(listed))])
|
||||
ProxyLogging._callback_capabilities_cache.clear()
|
||||
assert list(await _model_group_rows(caller)) == listed
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@_LISTING_GATES
|
||||
async def test_a_name_a_listing_callback_hides_stays_hidden_when_it_is_also_an_alias(
|
||||
router: Router, monkeypatch: pytest.MonkeyPatch, gate: type[_ListingGate | _V2ListingGate]
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "callbacks", [gate(frozenset({"fast"}))])
|
||||
ProxyLogging._callback_capabilities_cache.clear()
|
||||
models: Final = ["gpt-4.1-mini", "fast"]
|
||||
caller: Final = _team_member(models=models, team_model_aliases={"fast": "gpt-4.1-mini"})
|
||||
unaliased: Final = _team_member(models=models)
|
||||
|
||||
assert await _v1_models(caller) == await _v1_models(unaliased)
|
||||
assert await _model_group_rows(caller) == await _model_group_rows(unaliased)
|
||||
with pytest.raises(HTTPException) as retrieval:
|
||||
await proxy_server.model_info(model_id="fast", user_api_key_dict=caller)
|
||||
with pytest.raises(HTTPException) as unaliased_retrieval:
|
||||
await proxy_server.model_info(model_id="fast", user_api_key_dict=unaliased)
|
||||
assert retrieval.value.status_code == unaliased_retrieval.value.status_code
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@_LISTING_GATES
|
||||
async def test_an_alias_to_an_unlisted_model_a_listing_callback_keeps_is_described_by_that_model(
|
||||
router: Router, monkeypatch: pytest.MonkeyPatch, gate: type[_ListingGate | _V2ListingGate]
|
||||
) -> None:
|
||||
caller: Final = _team_listing("fast", team_model_aliases={"fast": "gpt-4.1"})
|
||||
target_holder: Final = _keyholder("gpt-4.1")
|
||||
target_row: Final = {row["id"]: row for row in (await proxy_server.model_list(user_api_key_dict=target_holder, request=None))["data"]}
|
||||
target_info: Final = await proxy_server.model_info(model_id="gpt-4.1", user_api_key_dict=target_holder)
|
||||
target_group: Final = {**(await _model_group_rows(target_holder))["gpt-4.1"], "model_group": "fast"}
|
||||
monkeypatch.setattr(litellm, "callbacks", [gate(frozenset({"unrelated"}))])
|
||||
ProxyLogging._callback_capabilities_cache.clear()
|
||||
|
||||
rows: Final = {row["id"]: row for row in (await proxy_server.model_list(user_api_key_dict=caller, request=None))["data"]}
|
||||
assert rows["fast"] == {**target_row["gpt-4.1"], "id": "fast"}
|
||||
assert await proxy_server.model_info(model_id="fast", user_api_key_dict=caller) == {**target_info, "id": "fast"}
|
||||
assert (await _model_group_rows(caller))["fast"] == target_group
|
||||
assert await _model_group_rows(caller, model_group="fast") == {"fast": target_group}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@_LISTING_GATES
|
||||
async def test_an_alias_to_an_unlisted_model_a_listing_callback_hides_is_not_described_by_that_model(
|
||||
router: Router, monkeypatch: pytest.MonkeyPatch, gate: type[_ListingGate | _V2ListingGate]
|
||||
) -> None:
|
||||
caller: Final = _team_listing("fast", team_model_aliases={"fast": "gpt-4.1"})
|
||||
monkeypatch.setattr(litellm, "callbacks", [gate(frozenset({"gpt-4.1"}))])
|
||||
ProxyLogging._callback_capabilities_cache.clear()
|
||||
|
||||
assert (await _model_group_rows(caller))["fast"]["providers"] == []
|
||||
with pytest.raises(HTTPException) as lookup:
|
||||
await proxy_server.model_info(model_id="fast", user_api_key_dict=caller)
|
||||
assert lookup.value.status_code == 404
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@_LISTING_GATES
|
||||
async def test_an_alias_named_like_a_router_alias_to_a_hidden_model_is_listed_as_its_own_target(
|
||||
router: Router, monkeypatch: pytest.MonkeyPatch, gate: type[_ListingGate | _V2ListingGate]
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "callbacks", [gate(frozenset({"gpt-4.1"}))])
|
||||
ProxyLogging._callback_capabilities_cache.clear()
|
||||
caller: Final = UserAPIKeyAuth(
|
||||
api_key="sk-test",
|
||||
user_id="u",
|
||||
user_role=LitellmUserRoles.INTERNAL_USER,
|
||||
models=["gpt-4.1-mini", "gpt-4.1"],
|
||||
aliases={"legacy": "gpt-4.1-mini"},
|
||||
)
|
||||
|
||||
listing: Final = await proxy_server.model_list(user_api_key_dict=caller, request=None)
|
||||
rows: Final = {row["id"]: row for row in listing["data"]}
|
||||
assert rows["legacy"] == {**rows["gpt-4.1-mini"], "id": "legacy"}
|
||||
target: Final = await proxy_server.model_info(model_id="gpt-4.1-mini", user_api_key_dict=caller)
|
||||
assert await proxy_server.model_info(model_id="legacy", user_api_key_dict=caller) == {**target, "id": "legacy"}
|
||||
groups: Final = await _model_group_rows(caller)
|
||||
assert groups["legacy"] == {**groups["gpt-4.1-mini"], "model_group": "legacy"}
|
||||
assert await _model_group_rows(caller, model_group="legacy") == {"legacy": groups["legacy"]}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(("model_group", "offered"), [("gpt-4.1-mini", {"gpt-4.1-mini"}), ("fast", {"gpt-4.1"})])
|
||||
async def test_model_group_info_for_one_group_offers_listing_callbacks_only_the_names_it_reads(
|
||||
router: Router, monkeypatch: pytest.MonkeyPatch, model_group: str, offered: set[str]
|
||||
) -> None:
|
||||
recorder: Final = _ListingRecorder()
|
||||
monkeypatch.setattr(litellm, "callbacks", [recorder])
|
||||
ProxyLogging._callback_capabilities_cache.clear()
|
||||
caller: Final = _team_member(
|
||||
models=["gpt-4.1-mini", "gpt-4.1", "model_name_team1_abc"], team_model_aliases={"fast": "gpt-4.1"}
|
||||
)
|
||||
|
||||
rows: Final = await _model_group_rows(caller, model_group=model_group)
|
||||
|
||||
assert list(rows) == [model_group]
|
||||
assert set(chain.from_iterable(recorder.offered)) == offered
|
||||
assert rows[model_group] == (await _model_group_rows(caller))[model_group]
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue