mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(health): show deployments reached through wildcard and alias allowlists, and by id when an alias shadows the name
This commit is contained in:
parent
e9e76077d9
commit
8d9eab02fc
3 changed files with 233 additions and 27 deletions
|
|
@ -7,7 +7,9 @@ import secrets
|
|||
import time
|
||||
import traceback
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from itertools import chain, groupby
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Literal, TypedDict
|
||||
|
||||
|
|
@ -81,6 +83,7 @@ from litellm.router_utils.clientside_credential_handler import (
|
|||
_ADMIN_CONFIG_FIELDS_TO_CLEAR_ON_BASE_OVERRIDE, # pyright: ignore[reportPrivateUsage] # one canonical list, shared with the router path
|
||||
clientside_credential_keys,
|
||||
)
|
||||
from litellm.router_utils.common_utils import resolve_model_group_alias
|
||||
from litellm.secret_managers.main import get_secret_bool
|
||||
from litellm.types.proxy.litellm_pre_call_utils import RedactedDict, SecretFields
|
||||
from litellm.types.router import reject_server_owned_wif_fields
|
||||
|
|
@ -255,7 +258,7 @@ def get_callback_identifier(callback):
|
|||
|
||||
router: Final = APIRouter()
|
||||
_OBJECT_MAPPING: Final = TypeAdapter(Mapping[str, object])
|
||||
_DEPLOYMENT_LIST: Final = TypeAdapter(tuple[object, ...])
|
||||
_OBJECTS: Final = TypeAdapter(tuple[object, ...])
|
||||
services = (
|
||||
Literal[
|
||||
"slack_budget_alerts",
|
||||
|
|
@ -1101,34 +1104,139 @@ async def _caller_aliases(user_api_key_dict: UserAPIKeyAuth) -> CallerAliases:
|
|||
)
|
||||
|
||||
|
||||
def _served_model_group(model_name: str, aliases: CallerAliases, llm_router: Router | None) -> str:
|
||||
def _model_auth_checks(model_name: str, aliases: CallerAliases) -> str:
|
||||
return resolve_model_group_alias(aliases.router, model_name) or model_name
|
||||
|
||||
|
||||
def _deployment_id(deployment: object) -> str | None:
|
||||
match deployment:
|
||||
case {"model_info": {"id": str(model_id)}}:
|
||||
return model_id
|
||||
case _:
|
||||
return None
|
||||
|
||||
|
||||
def _granted_names(user_api_key_dict: UserAPIKeyAuth) -> frozenset[str]:
|
||||
grants: Final = (
|
||||
*_OBJECTS.validate_python(user_api_key_dict.models),
|
||||
*_OBJECTS.validate_python(user_api_key_dict.team_models),
|
||||
)
|
||||
return frozenset(name for name in grants if isinstance(name, str))
|
||||
|
||||
|
||||
def _served_group(model_name: str, aliases: CallerAliases, llm_router: Router | None) -> str:
|
||||
dispatched: Final = dispatched_model(model_name, aliases)
|
||||
return llm_router.routable_model_group(dispatched) if llm_router is not None else dispatched
|
||||
|
||||
|
||||
async def _model_groups_key_may_reach(
|
||||
llm_model_list: Sequence[object], user_api_key_dict: UserAPIKeyAuth, llm_router: Router | None
|
||||
) -> frozenset[str]:
|
||||
requested_names: Final = frozenset[str]().union(
|
||||
*(_names_caller_may_request(deployment, user_api_key_dict.team_id) for deployment in llm_model_list)
|
||||
def _name_id_pairs(deployment: object, team_id: str | None) -> tuple[tuple[str, str], ...]:
|
||||
model_id: Final = _deployment_id(deployment)
|
||||
if model_id is None:
|
||||
return ()
|
||||
return tuple((name, model_id) for name in _names_caller_may_request(deployment, team_id) | {model_id})
|
||||
|
||||
|
||||
def _ids_by_requested_name(llm_model_list: Sequence[object], team_id: str | None) -> Mapping[str, frozenset[str]]:
|
||||
pairs: Final = sorted(chain.from_iterable(_name_id_pairs(deployment, team_id) for deployment in llm_model_list))
|
||||
return MappingProxyType(
|
||||
{name: frozenset(model_id for _, model_id in group) for name, group in groupby(pairs, key=lambda pair: pair[0])}
|
||||
)
|
||||
aliases: Final = await _caller_aliases(user_api_key_dict)
|
||||
|
||||
|
||||
def _ids_serving_group(
|
||||
group: str,
|
||||
ids_by_name: Mapping[str, frozenset[str]],
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
llm_router: Router | None,
|
||||
) -> frozenset[str]:
|
||||
named: Final = ids_by_name.get(group)
|
||||
if named is not None or llm_router is None:
|
||||
return named or frozenset()
|
||||
routed: Final = llm_router.get_routable_deployments(
|
||||
group, {"metadata": {"user_api_key_team_id": user_api_key_dict.team_id}}
|
||||
)
|
||||
return frozenset(model_id for model_id in map(_deployment_id, routed) if model_id is not None)
|
||||
|
||||
|
||||
async def _names_key_may_request(
|
||||
names: Iterable[str], aliases: CallerAliases, user_api_key_dict: UserAPIKeyAuth, llm_router: Router | None
|
||||
) -> frozenset[str]:
|
||||
return frozenset(
|
||||
[
|
||||
_served_model_group(name, aliases, llm_router)
|
||||
for name in requested_names
|
||||
if await _key_may_call_model(name, user_api_key_dict, llm_router)
|
||||
name
|
||||
for name in names
|
||||
if await _key_may_call_model(_model_auth_checks(name, aliases), user_api_key_dict, llm_router)
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def _ids_serving(
|
||||
groups: Iterable[str],
|
||||
ids_by_name: Mapping[str, frozenset[str]],
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
llm_router: Router | None,
|
||||
) -> frozenset[str]:
|
||||
return frozenset[str]().union(
|
||||
*(_ids_serving_group(group, ids_by_name, user_api_key_dict, llm_router) for group in groups)
|
||||
)
|
||||
|
||||
|
||||
def _shadowed_ids(
|
||||
llm_model_list: Sequence[object], admitted_names: frozenset[str], reached_ids: frozenset[str], team_id: str | None
|
||||
) -> frozenset[str]:
|
||||
ids_of_admitted_names: Final = (
|
||||
_deployment_id(deployment)
|
||||
for deployment in llm_model_list
|
||||
if not admitted_names.isdisjoint(_names_caller_may_request(deployment, team_id))
|
||||
)
|
||||
return frozenset(model_id for model_id in ids_of_admitted_names if model_id is not None) - reached_ids
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _ReachableDeployments:
|
||||
model_ids: frozenset[str]
|
||||
groups: frozenset[str]
|
||||
|
||||
def include(self, deployment: object, team_id: str | None) -> bool:
|
||||
model_id: Final = _deployment_id(deployment)
|
||||
if model_id is None:
|
||||
return not self.groups.isdisjoint(_names_caller_may_request(deployment, team_id))
|
||||
return model_id in self.model_ids
|
||||
|
||||
|
||||
_NOTHING_REACHABLE: Final = _ReachableDeployments(frozenset(), frozenset())
|
||||
|
||||
|
||||
async def _deployments_key_may_reach(
|
||||
llm_model_list: Sequence[object], user_api_key_dict: UserAPIKeyAuth, llm_router: Router | None
|
||||
) -> _ReachableDeployments:
|
||||
aliases: Final = await _caller_aliases(user_api_key_dict)
|
||||
candidate_names: Final = _granted_names(user_api_key_dict).union(
|
||||
*(_names_caller_may_request(deployment, user_api_key_dict.team_id) for deployment in llm_model_list)
|
||||
)
|
||||
admitted_names: Final = await _names_key_may_request(candidate_names, aliases, user_api_key_dict, llm_router)
|
||||
groups: Final = frozenset(_served_group(name, aliases, llm_router) for name in admitted_names)
|
||||
ids_by_name: Final = _ids_by_requested_name(llm_model_list, user_api_key_dict.team_id)
|
||||
reached_ids: Final = _ids_serving(groups, ids_by_name, user_api_key_dict, llm_router)
|
||||
admitted_ids: Final = await _names_key_may_request(
|
||||
_shadowed_ids(llm_model_list, admitted_names, reached_ids, user_api_key_dict.team_id),
|
||||
aliases,
|
||||
user_api_key_dict,
|
||||
llm_router,
|
||||
)
|
||||
id_groups: Final = frozenset(_served_group(model_id, aliases, llm_router) for model_id in admitted_ids)
|
||||
return _ReachableDeployments(
|
||||
reached_ids | _ids_serving(id_groups, ids_by_name, user_api_key_dict, llm_router), groups
|
||||
)
|
||||
|
||||
|
||||
def _caller_may_probe_deployment(
|
||||
deployment: Mapping[str, object], reachable_groups: frozenset[str], team_id: str | None, caller_is_admin: bool
|
||||
deployment: Mapping[str, object], reachable: _ReachableDeployments, team_id: str | None, caller_is_admin: bool
|
||||
) -> bool:
|
||||
"""Same deployment visibility rule as routing: another team's deployment is never in scope, team-less callers included."""
|
||||
if not caller_is_admin and not Router.deployment_usable_by_team(deployment, team_id):
|
||||
return False
|
||||
return not reachable_groups.isdisjoint(_names_caller_may_request(deployment, team_id))
|
||||
return reachable.include(deployment, team_id)
|
||||
|
||||
|
||||
def _resolve_targeted_model_ids(
|
||||
|
|
@ -1353,18 +1461,16 @@ async def health_endpoint(
|
|||
detail={"error": "Model list not initialized"},
|
||||
)
|
||||
restrict_to_allowed_models: Final = not is_admin or not _key_models_unrestricted(user_api_key_dict)
|
||||
reachable_groups: Final = (
|
||||
await _model_groups_key_may_reach(
|
||||
_DEPLOYMENT_LIST.validate_python(llm_model_list), user_api_key_dict, llm_router
|
||||
)
|
||||
reachable: Final = (
|
||||
await _deployments_key_may_reach(_OBJECTS.validate_python(llm_model_list), user_api_key_dict, llm_router)
|
||||
if restrict_to_allowed_models
|
||||
else frozenset[str]()
|
||||
else _NOTHING_REACHABLE
|
||||
)
|
||||
_llm_model_list: Final = [
|
||||
m
|
||||
for m in copy.deepcopy(llm_model_list)
|
||||
if not restrict_to_allowed_models
|
||||
or _caller_may_probe_deployment(m, reachable_groups, user_api_key_dict.team_id, is_admin)
|
||||
or _caller_may_probe_deployment(m, reachable, user_api_key_dict.team_id, is_admin)
|
||||
]
|
||||
targeted_ids: Final = _resolve_targeted_model_ids(_llm_model_list, model, model_id, user_api_key_dict.team_id)
|
||||
if restrict_to_allowed_models and targeted_ids is not None and not targeted_ids:
|
||||
|
|
|
|||
|
|
@ -14694,9 +14694,11 @@ class Router:
|
|||
"""
|
||||
return [m for m in self.model_list if m["litellm_params"]["model"] == model]
|
||||
|
||||
def get_routable_deployments(self, model: str) -> tuple[DeploymentTypedDict, ...]:
|
||||
def get_routable_deployments(
|
||||
self, model: str, request_kwargs: Mapping[str, object] | None = None
|
||||
) -> tuple[DeploymentTypedDict, ...]:
|
||||
try:
|
||||
_, candidates = self._common_checks_available_deployment(model=model, request_kwargs={}) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # its signature leaves input and the result untyped
|
||||
_, candidates = self._common_checks_available_deployment(model=model, request_kwargs=request_kwargs or {}) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # its signature leaves input and the result untyped
|
||||
except (litellm.BadRequestError, ValueError):
|
||||
return ()
|
||||
if isinstance(candidates, dict):
|
||||
|
|
|
|||
|
|
@ -4289,7 +4289,9 @@ _RESTRICTED_TEAM_KEY = UserAPIKeyAuth(
|
|||
("caller", "expected_ids"),
|
||||
[(_PROVIDER_WILDCARD_KEY, {"id-prefixed-nano", "id-mini"}), (_RESTRICTED_TEAM_KEY, {"id-mini"})],
|
||||
)
|
||||
async def test_health_endpoint_probes_the_deployments_auth_lets_the_key_request(caller, expected_ids):
|
||||
async def test_health_endpoint_probes_the_deployments_auth_lets_the_key_request(
|
||||
caller: UserAPIKeyAuth, expected_ids: set[str]
|
||||
):
|
||||
probed = await _live_probed_model_ids(_PROVIDER_PREFIXED_MODEL_LIST, caller)
|
||||
|
||||
assert probed == expected_ids
|
||||
|
|
@ -4301,7 +4303,7 @@ async def test_health_endpoint_probes_the_deployments_auth_lets_the_key_request(
|
|||
[(_PROVIDER_WILDCARD_KEY, ["id-prefixed-nano", "id-mini"]), (_RESTRICTED_TEAM_KEY, ["id-mini"])],
|
||||
)
|
||||
async def test_health_endpoint_returns_the_deployments_auth_lets_the_key_request_on_background_cache_path(
|
||||
caller, expected_ids
|
||||
caller: UserAPIKeyAuth, expected_ids: list[str]
|
||||
):
|
||||
from fastapi import Response
|
||||
|
||||
|
|
@ -4320,7 +4322,7 @@ async def test_health_endpoint_returns_the_deployments_auth_lets_the_key_request
|
|||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("use_background_health_checks", [False, True])
|
||||
async def test_health_endpoint_refuses_a_deployment_the_team_allowlist_forbids(use_background_health_checks):
|
||||
async def test_health_endpoint_refuses_a_deployment_the_team_allowlist_forbids(use_background_health_checks: bool):
|
||||
from fastapi import HTTPException, Response
|
||||
|
||||
from litellm.proxy.health_endpoints._health_endpoints import health_endpoint
|
||||
|
|
@ -4369,7 +4371,7 @@ def _router_with_aliases(model_group_alias: Mapping[str, str]) -> Router:
|
|||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(("caller", "model_group_alias"), _ALIAS_SHADOW_CASES)
|
||||
async def test_health_endpoint_probes_the_deployment_an_alias_sends_the_key_to_not_the_one_it_shadows(
|
||||
caller, model_group_alias
|
||||
caller: UserAPIKeyAuth, model_group_alias: Mapping[str, str]
|
||||
):
|
||||
probed = await _live_probed_model_ids(
|
||||
_PROVIDER_PREFIXED_MODEL_LIST, caller, llm_router=_router_with_aliases(model_group_alias)
|
||||
|
|
@ -4381,7 +4383,7 @@ async def test_health_endpoint_probes_the_deployment_an_alias_sends_the_key_to_n
|
|||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(("caller", "model_group_alias"), _ALIAS_SHADOW_CASES)
|
||||
async def test_health_endpoint_returns_the_deployment_an_alias_sends_the_key_to_on_background_cache_path(
|
||||
caller, model_group_alias
|
||||
caller: UserAPIKeyAuth, model_group_alias: Mapping[str, str]
|
||||
):
|
||||
from fastapi import Response
|
||||
|
||||
|
|
@ -4400,7 +4402,7 @@ async def test_health_endpoint_returns_the_deployment_an_alias_sends_the_key_to_
|
|||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("use_background_health_checks", [False, True])
|
||||
async def test_health_endpoint_refuses_the_deployment_a_key_alias_shadows(use_background_health_checks):
|
||||
async def test_health_endpoint_refuses_the_deployment_a_key_alias_shadows(use_background_health_checks: bool):
|
||||
from fastapi import HTTPException, Response
|
||||
|
||||
from litellm.proxy.health_endpoints._health_endpoints import health_endpoint
|
||||
|
|
@ -4438,6 +4440,102 @@ async def test_health_endpoint_never_probes_a_deployment_without_a_model_name_fo
|
|||
assert probed == {"id-mini"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_health_endpoint_never_probes_the_deployment_a_keys_router_alias_sends_it_to_but_forbids():
|
||||
caller = UserAPIKeyAuth(
|
||||
api_key="hashed-test-key",
|
||||
models=["gpt-5.4-mini", "openai/gpt-5.4-nano"],
|
||||
router_settings={"model_group_alias": {"gpt-5.4-mini": "claude-haiku-5-5"}},
|
||||
)
|
||||
|
||||
probed = await _live_probed_model_ids(_PROVIDER_PREFIXED_MODEL_LIST, caller)
|
||||
|
||||
assert probed == {"id-prefixed-nano"}
|
||||
|
||||
|
||||
_WILDCARD_DEPLOYMENT = {"model_name": "openai/*", "litellm_params": {"model": "openai/*"}, "model_info": {"id": "id-wild"}}
|
||||
_CATCH_ALL_DEPLOYMENT = {"model_name": "*", "litellm_params": {"model": "*"}, "model_info": {"id": "id-star"}}
|
||||
_TEAM_WILDCARD_DEPLOYMENT = {
|
||||
"model_name": "openai-wild_team-a_7c1e",
|
||||
"litellm_params": {"model": "openai/*"},
|
||||
"model_info": {"id": "id-team-wild", "team_id": "team-a", "team_public_model_name": "openai/*"},
|
||||
}
|
||||
|
||||
|
||||
def _team_key(team_models: list[str], team_model_aliases: Mapping[str, str] | None = None) -> UserAPIKeyAuth:
|
||||
return UserAPIKeyAuth(
|
||||
api_key="hashed-test-key",
|
||||
models=[],
|
||||
team_id="team-a",
|
||||
team_models=team_models,
|
||||
team_model_aliases=dict(team_model_aliases) if team_model_aliases is not None else None,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("model_list", "model_group_alias", "caller", "expected_ids"),
|
||||
[
|
||||
pytest.param(
|
||||
[_WILDCARD_DEPLOYMENT, _PROVIDER_PREFIXED_MODEL_LIST[2]],
|
||||
{},
|
||||
_team_key(["openai/gpt-5.4-nano"]),
|
||||
{"id-wild"},
|
||||
id="team-name-served-by-provider-wildcard",
|
||||
),
|
||||
pytest.param(
|
||||
[_TEAM_WILDCARD_DEPLOYMENT, _PROVIDER_PREFIXED_MODEL_LIST[2]],
|
||||
{},
|
||||
_team_key(["openai/gpt-5.4-nano"]),
|
||||
{"id-team-wild"},
|
||||
id="team-name-served-by-teams-own-wildcard",
|
||||
),
|
||||
pytest.param(
|
||||
[_CATCH_ALL_DEPLOYMENT],
|
||||
{},
|
||||
_team_key(["gpt-5.4-mini"]),
|
||||
{"id-star"},
|
||||
id="team-name-served-by-catch-all",
|
||||
),
|
||||
pytest.param(
|
||||
_PROVIDER_PREFIXED_MODEL_LIST,
|
||||
{"gpt-4": "gpt-5.4-mini"},
|
||||
_team_key(["gpt-4"]),
|
||||
{"id-mini"},
|
||||
id="team-allows-router-alias",
|
||||
),
|
||||
pytest.param(
|
||||
_PROVIDER_PREFIXED_MODEL_LIST,
|
||||
{},
|
||||
_team_key(["my-alias"], {"my-alias": "gpt-5.4-mini"}),
|
||||
{"id-mini"},
|
||||
id="team-allows-team-alias",
|
||||
),
|
||||
pytest.param(
|
||||
_PROVIDER_PREFIXED_MODEL_LIST,
|
||||
{"claude-haiku-5-5": "gpt-5.4-mini"},
|
||||
UserAPIKeyAuth(api_key="hashed-test-key", models=[]),
|
||||
{"id-prefixed-nano", "id-mini", "id-haiku"},
|
||||
id="unrestricted-key-calls-shadowed-deployment-by-id",
|
||||
),
|
||||
],
|
||||
)
|
||||
async def test_health_endpoint_probes_every_deployment_the_callers_allowed_names_route_to(
|
||||
model_list: Sequence[Mapping[str, object]],
|
||||
model_group_alias: Mapping[str, str],
|
||||
caller: UserAPIKeyAuth,
|
||||
expected_ids: set[str],
|
||||
):
|
||||
llm_router = Router(
|
||||
model_list=copy.deepcopy([dict(deployment) for deployment in model_list]),
|
||||
model_group_alias=dict(model_group_alias),
|
||||
)
|
||||
|
||||
probed = await _live_probed_model_ids(model_list, caller, llm_router=llm_router)
|
||||
|
||||
assert probed == expected_ids
|
||||
|
||||
|
||||
def test_resolve_targeted_model_ids_lets_model_id_win_over_model():
|
||||
resolve = _health_endpoints_module._resolve_targeted_model_ids
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue