diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index fefa1acf32e..f9e8fc69824 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -7,7 +7,9 @@ import secrets import time import traceback from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass from datetime import datetime, timedelta, timezone +from itertools import chain, groupby from types import MappingProxyType from typing import Final, Literal, TypedDict @@ -81,6 +83,7 @@ from litellm.router_utils.clientside_credential_handler import ( _ADMIN_CONFIG_FIELDS_TO_CLEAR_ON_BASE_OVERRIDE, # pyright: ignore[reportPrivateUsage] # one canonical list, shared with the router path clientside_credential_keys, ) +from litellm.router_utils.common_utils import resolve_model_group_alias from litellm.secret_managers.main import get_secret_bool from litellm.types.proxy.litellm_pre_call_utils import RedactedDict, SecretFields from litellm.types.router import reject_server_owned_wif_fields @@ -255,7 +258,7 @@ def get_callback_identifier(callback): router: Final = APIRouter() _OBJECT_MAPPING: Final = TypeAdapter(Mapping[str, object]) -_DEPLOYMENT_LIST: Final = TypeAdapter(tuple[object, ...]) +_OBJECTS: Final = TypeAdapter(tuple[object, ...]) services = ( Literal[ "slack_budget_alerts", @@ -1101,34 +1104,139 @@ async def _caller_aliases(user_api_key_dict: UserAPIKeyAuth) -> CallerAliases: ) -def _served_model_group(model_name: str, aliases: CallerAliases, llm_router: Router | None) -> str: +def _model_auth_checks(model_name: str, aliases: CallerAliases) -> str: + return resolve_model_group_alias(aliases.router, model_name) or model_name + + +def _deployment_id(deployment: object) -> str | None: + match deployment: + case {"model_info": {"id": str(model_id)}}: + return model_id + case _: + return None + + +def _granted_names(user_api_key_dict: UserAPIKeyAuth) -> frozenset[str]: + grants: Final = ( + *_OBJECTS.validate_python(user_api_key_dict.models), + *_OBJECTS.validate_python(user_api_key_dict.team_models), + ) + return frozenset(name for name in grants if isinstance(name, str)) + + +def _served_group(model_name: str, aliases: CallerAliases, llm_router: Router | None) -> str: dispatched: Final = dispatched_model(model_name, aliases) return llm_router.routable_model_group(dispatched) if llm_router is not None else dispatched -async def _model_groups_key_may_reach( - llm_model_list: Sequence[object], user_api_key_dict: UserAPIKeyAuth, llm_router: Router | None -) -> frozenset[str]: - requested_names: Final = frozenset[str]().union( - *(_names_caller_may_request(deployment, user_api_key_dict.team_id) for deployment in llm_model_list) +def _name_id_pairs(deployment: object, team_id: str | None) -> tuple[tuple[str, str], ...]: + model_id: Final = _deployment_id(deployment) + if model_id is None: + return () + return tuple((name, model_id) for name in _names_caller_may_request(deployment, team_id) | {model_id}) + + +def _ids_by_requested_name(llm_model_list: Sequence[object], team_id: str | None) -> Mapping[str, frozenset[str]]: + pairs: Final = sorted(chain.from_iterable(_name_id_pairs(deployment, team_id) for deployment in llm_model_list)) + return MappingProxyType( + {name: frozenset(model_id for _, model_id in group) for name, group in groupby(pairs, key=lambda pair: pair[0])} ) - aliases: Final = await _caller_aliases(user_api_key_dict) + + +def _ids_serving_group( + group: str, + ids_by_name: Mapping[str, frozenset[str]], + user_api_key_dict: UserAPIKeyAuth, + llm_router: Router | None, +) -> frozenset[str]: + named: Final = ids_by_name.get(group) + if named is not None or llm_router is None: + return named or frozenset() + routed: Final = llm_router.get_routable_deployments( + group, {"metadata": {"user_api_key_team_id": user_api_key_dict.team_id}} + ) + return frozenset(model_id for model_id in map(_deployment_id, routed) if model_id is not None) + + +async def _names_key_may_request( + names: Iterable[str], aliases: CallerAliases, user_api_key_dict: UserAPIKeyAuth, llm_router: Router | None +) -> frozenset[str]: return frozenset( [ - _served_model_group(name, aliases, llm_router) - for name in requested_names - if await _key_may_call_model(name, user_api_key_dict, llm_router) + name + for name in names + if await _key_may_call_model(_model_auth_checks(name, aliases), user_api_key_dict, llm_router) ] ) +def _ids_serving( + groups: Iterable[str], + ids_by_name: Mapping[str, frozenset[str]], + user_api_key_dict: UserAPIKeyAuth, + llm_router: Router | None, +) -> frozenset[str]: + return frozenset[str]().union( + *(_ids_serving_group(group, ids_by_name, user_api_key_dict, llm_router) for group in groups) + ) + + +def _shadowed_ids( + llm_model_list: Sequence[object], admitted_names: frozenset[str], reached_ids: frozenset[str], team_id: str | None +) -> frozenset[str]: + ids_of_admitted_names: Final = ( + _deployment_id(deployment) + for deployment in llm_model_list + if not admitted_names.isdisjoint(_names_caller_may_request(deployment, team_id)) + ) + return frozenset(model_id for model_id in ids_of_admitted_names if model_id is not None) - reached_ids + + +@dataclass(frozen=True, slots=True) +class _ReachableDeployments: + model_ids: frozenset[str] + groups: frozenset[str] + + def include(self, deployment: object, team_id: str | None) -> bool: + model_id: Final = _deployment_id(deployment) + if model_id is None: + return not self.groups.isdisjoint(_names_caller_may_request(deployment, team_id)) + return model_id in self.model_ids + + +_NOTHING_REACHABLE: Final = _ReachableDeployments(frozenset(), frozenset()) + + +async def _deployments_key_may_reach( + llm_model_list: Sequence[object], user_api_key_dict: UserAPIKeyAuth, llm_router: Router | None +) -> _ReachableDeployments: + aliases: Final = await _caller_aliases(user_api_key_dict) + candidate_names: Final = _granted_names(user_api_key_dict).union( + *(_names_caller_may_request(deployment, user_api_key_dict.team_id) for deployment in llm_model_list) + ) + admitted_names: Final = await _names_key_may_request(candidate_names, aliases, user_api_key_dict, llm_router) + groups: Final = frozenset(_served_group(name, aliases, llm_router) for name in admitted_names) + ids_by_name: Final = _ids_by_requested_name(llm_model_list, user_api_key_dict.team_id) + reached_ids: Final = _ids_serving(groups, ids_by_name, user_api_key_dict, llm_router) + admitted_ids: Final = await _names_key_may_request( + _shadowed_ids(llm_model_list, admitted_names, reached_ids, user_api_key_dict.team_id), + aliases, + user_api_key_dict, + llm_router, + ) + id_groups: Final = frozenset(_served_group(model_id, aliases, llm_router) for model_id in admitted_ids) + return _ReachableDeployments( + reached_ids | _ids_serving(id_groups, ids_by_name, user_api_key_dict, llm_router), groups + ) + + def _caller_may_probe_deployment( - deployment: Mapping[str, object], reachable_groups: frozenset[str], team_id: str | None, caller_is_admin: bool + deployment: Mapping[str, object], reachable: _ReachableDeployments, team_id: str | None, caller_is_admin: bool ) -> bool: """Same deployment visibility rule as routing: another team's deployment is never in scope, team-less callers included.""" if not caller_is_admin and not Router.deployment_usable_by_team(deployment, team_id): return False - return not reachable_groups.isdisjoint(_names_caller_may_request(deployment, team_id)) + return reachable.include(deployment, team_id) def _resolve_targeted_model_ids( @@ -1353,18 +1461,16 @@ async def health_endpoint( detail={"error": "Model list not initialized"}, ) restrict_to_allowed_models: Final = not is_admin or not _key_models_unrestricted(user_api_key_dict) - reachable_groups: Final = ( - await _model_groups_key_may_reach( - _DEPLOYMENT_LIST.validate_python(llm_model_list), user_api_key_dict, llm_router - ) + reachable: Final = ( + await _deployments_key_may_reach(_OBJECTS.validate_python(llm_model_list), user_api_key_dict, llm_router) if restrict_to_allowed_models - else frozenset[str]() + else _NOTHING_REACHABLE ) _llm_model_list: Final = [ m for m in copy.deepcopy(llm_model_list) if not restrict_to_allowed_models - or _caller_may_probe_deployment(m, reachable_groups, user_api_key_dict.team_id, is_admin) + or _caller_may_probe_deployment(m, reachable, user_api_key_dict.team_id, is_admin) ] targeted_ids: Final = _resolve_targeted_model_ids(_llm_model_list, model, model_id, user_api_key_dict.team_id) if restrict_to_allowed_models and targeted_ids is not None and not targeted_ids: diff --git a/litellm/router.py b/litellm/router.py index 5b18d36316f..a96419c739f 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -14694,9 +14694,11 @@ class Router: """ return [m for m in self.model_list if m["litellm_params"]["model"] == model] - def get_routable_deployments(self, model: str) -> tuple[DeploymentTypedDict, ...]: + def get_routable_deployments( + self, model: str, request_kwargs: Mapping[str, object] | None = None + ) -> tuple[DeploymentTypedDict, ...]: try: - _, candidates = self._common_checks_available_deployment(model=model, request_kwargs={}) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # its signature leaves input and the result untyped + _, candidates = self._common_checks_available_deployment(model=model, request_kwargs=request_kwargs or {}) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # its signature leaves input and the result untyped except (litellm.BadRequestError, ValueError): return () if isinstance(candidates, dict): diff --git a/tests/unit/proxy/health_endpoints/test_health_endpoints.py b/tests/unit/proxy/health_endpoints/test_health_endpoints.py index c97c1d0805e..37b97079905 100644 --- a/tests/unit/proxy/health_endpoints/test_health_endpoints.py +++ b/tests/unit/proxy/health_endpoints/test_health_endpoints.py @@ -4289,7 +4289,9 @@ _RESTRICTED_TEAM_KEY = UserAPIKeyAuth( ("caller", "expected_ids"), [(_PROVIDER_WILDCARD_KEY, {"id-prefixed-nano", "id-mini"}), (_RESTRICTED_TEAM_KEY, {"id-mini"})], ) -async def test_health_endpoint_probes_the_deployments_auth_lets_the_key_request(caller, expected_ids): +async def test_health_endpoint_probes_the_deployments_auth_lets_the_key_request( + caller: UserAPIKeyAuth, expected_ids: set[str] +): probed = await _live_probed_model_ids(_PROVIDER_PREFIXED_MODEL_LIST, caller) assert probed == expected_ids @@ -4301,7 +4303,7 @@ async def test_health_endpoint_probes_the_deployments_auth_lets_the_key_request( [(_PROVIDER_WILDCARD_KEY, ["id-prefixed-nano", "id-mini"]), (_RESTRICTED_TEAM_KEY, ["id-mini"])], ) async def test_health_endpoint_returns_the_deployments_auth_lets_the_key_request_on_background_cache_path( - caller, expected_ids + caller: UserAPIKeyAuth, expected_ids: list[str] ): from fastapi import Response @@ -4320,7 +4322,7 @@ async def test_health_endpoint_returns_the_deployments_auth_lets_the_key_request @pytest.mark.asyncio @pytest.mark.parametrize("use_background_health_checks", [False, True]) -async def test_health_endpoint_refuses_a_deployment_the_team_allowlist_forbids(use_background_health_checks): +async def test_health_endpoint_refuses_a_deployment_the_team_allowlist_forbids(use_background_health_checks: bool): from fastapi import HTTPException, Response from litellm.proxy.health_endpoints._health_endpoints import health_endpoint @@ -4369,7 +4371,7 @@ def _router_with_aliases(model_group_alias: Mapping[str, str]) -> Router: @pytest.mark.asyncio @pytest.mark.parametrize(("caller", "model_group_alias"), _ALIAS_SHADOW_CASES) async def test_health_endpoint_probes_the_deployment_an_alias_sends_the_key_to_not_the_one_it_shadows( - caller, model_group_alias + caller: UserAPIKeyAuth, model_group_alias: Mapping[str, str] ): probed = await _live_probed_model_ids( _PROVIDER_PREFIXED_MODEL_LIST, caller, llm_router=_router_with_aliases(model_group_alias) @@ -4381,7 +4383,7 @@ async def test_health_endpoint_probes_the_deployment_an_alias_sends_the_key_to_n @pytest.mark.asyncio @pytest.mark.parametrize(("caller", "model_group_alias"), _ALIAS_SHADOW_CASES) async def test_health_endpoint_returns_the_deployment_an_alias_sends_the_key_to_on_background_cache_path( - caller, model_group_alias + caller: UserAPIKeyAuth, model_group_alias: Mapping[str, str] ): from fastapi import Response @@ -4400,7 +4402,7 @@ async def test_health_endpoint_returns_the_deployment_an_alias_sends_the_key_to_ @pytest.mark.asyncio @pytest.mark.parametrize("use_background_health_checks", [False, True]) -async def test_health_endpoint_refuses_the_deployment_a_key_alias_shadows(use_background_health_checks): +async def test_health_endpoint_refuses_the_deployment_a_key_alias_shadows(use_background_health_checks: bool): from fastapi import HTTPException, Response from litellm.proxy.health_endpoints._health_endpoints import health_endpoint @@ -4438,6 +4440,102 @@ async def test_health_endpoint_never_probes_a_deployment_without_a_model_name_fo assert probed == {"id-mini"} +@pytest.mark.asyncio +async def test_health_endpoint_never_probes_the_deployment_a_keys_router_alias_sends_it_to_but_forbids(): + caller = UserAPIKeyAuth( + api_key="hashed-test-key", + models=["gpt-5.4-mini", "openai/gpt-5.4-nano"], + router_settings={"model_group_alias": {"gpt-5.4-mini": "claude-haiku-5-5"}}, + ) + + probed = await _live_probed_model_ids(_PROVIDER_PREFIXED_MODEL_LIST, caller) + + assert probed == {"id-prefixed-nano"} + + +_WILDCARD_DEPLOYMENT = {"model_name": "openai/*", "litellm_params": {"model": "openai/*"}, "model_info": {"id": "id-wild"}} +_CATCH_ALL_DEPLOYMENT = {"model_name": "*", "litellm_params": {"model": "*"}, "model_info": {"id": "id-star"}} +_TEAM_WILDCARD_DEPLOYMENT = { + "model_name": "openai-wild_team-a_7c1e", + "litellm_params": {"model": "openai/*"}, + "model_info": {"id": "id-team-wild", "team_id": "team-a", "team_public_model_name": "openai/*"}, +} + + +def _team_key(team_models: list[str], team_model_aliases: Mapping[str, str] | None = None) -> UserAPIKeyAuth: + return UserAPIKeyAuth( + api_key="hashed-test-key", + models=[], + team_id="team-a", + team_models=team_models, + team_model_aliases=dict(team_model_aliases) if team_model_aliases is not None else None, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("model_list", "model_group_alias", "caller", "expected_ids"), + [ + pytest.param( + [_WILDCARD_DEPLOYMENT, _PROVIDER_PREFIXED_MODEL_LIST[2]], + {}, + _team_key(["openai/gpt-5.4-nano"]), + {"id-wild"}, + id="team-name-served-by-provider-wildcard", + ), + pytest.param( + [_TEAM_WILDCARD_DEPLOYMENT, _PROVIDER_PREFIXED_MODEL_LIST[2]], + {}, + _team_key(["openai/gpt-5.4-nano"]), + {"id-team-wild"}, + id="team-name-served-by-teams-own-wildcard", + ), + pytest.param( + [_CATCH_ALL_DEPLOYMENT], + {}, + _team_key(["gpt-5.4-mini"]), + {"id-star"}, + id="team-name-served-by-catch-all", + ), + pytest.param( + _PROVIDER_PREFIXED_MODEL_LIST, + {"gpt-4": "gpt-5.4-mini"}, + _team_key(["gpt-4"]), + {"id-mini"}, + id="team-allows-router-alias", + ), + pytest.param( + _PROVIDER_PREFIXED_MODEL_LIST, + {}, + _team_key(["my-alias"], {"my-alias": "gpt-5.4-mini"}), + {"id-mini"}, + id="team-allows-team-alias", + ), + pytest.param( + _PROVIDER_PREFIXED_MODEL_LIST, + {"claude-haiku-5-5": "gpt-5.4-mini"}, + UserAPIKeyAuth(api_key="hashed-test-key", models=[]), + {"id-prefixed-nano", "id-mini", "id-haiku"}, + id="unrestricted-key-calls-shadowed-deployment-by-id", + ), + ], +) +async def test_health_endpoint_probes_every_deployment_the_callers_allowed_names_route_to( + model_list: Sequence[Mapping[str, object]], + model_group_alias: Mapping[str, str], + caller: UserAPIKeyAuth, + expected_ids: set[str], +): + llm_router = Router( + model_list=copy.deepcopy([dict(deployment) for deployment in model_list]), + model_group_alias=dict(model_group_alias), + ) + + probed = await _live_probed_model_ids(model_list, caller, llm_router=llm_router) + + assert probed == expected_ids + + def test_resolve_targeted_model_ids_lets_model_id_win_over_model(): resolve = _health_endpoints_module._resolve_targeted_model_ids