diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 448643ef59b..890276eb696 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -45,6 +45,7 @@ from litellm.constants import ( ) from litellm.litellm_core_utils.dd_tracing import tracer from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider +from litellm.litellm_core_utils.resource_ownership import is_proxy_admin from litellm.litellm_core_utils.safe_json_loads import safe_json_loads from litellm.models.project import LiteLLM_ProjectTable from litellm.proxy._types import ( @@ -5503,7 +5504,7 @@ async def can_key_call_model( """ key_models: Final = _resolve_key_models_for_auth_check(valid_token=valid_token) try: - return can_object_call_model( + can_object_call_model( model=model, llm_router=llm_router, models=key_models, @@ -5515,22 +5516,50 @@ async def can_key_call_model( except ProxyException: # Fallback: check key's access_group_ids key_access_group_ids: Final = valid_token.access_group_ids or [] - if key_access_group_ids: - models_from_groups: Final = await get_models_from_access_groups( - access_group_ids=key_access_group_ids, - prisma_client=prisma_client, - ) - if models_from_groups: - return can_object_call_model( - model=model, - llm_router=llm_router, - models=models_from_groups, - team_model_aliases=valid_token.team_model_aliases, - team_id=valid_token.team_id, - key_model_aliases=key_model_aliases_for_auth_check(valid_token), - object_type="key", - ) - raise + if not key_access_group_ids: + raise + models_from_groups: Final = await get_models_from_access_groups( + access_group_ids=key_access_group_ids, + prisma_client=prisma_client, + ) + if not models_from_groups: + raise + can_object_call_model( + model=model, + llm_router=llm_router, + models=models_from_groups, + team_model_aliases=valid_token.team_model_aliases, + team_id=valid_token.team_id, + key_model_aliases=key_model_aliases_for_auth_check(valid_token), + object_type="key", + ) + raise_when_key_reaches_only_other_teams_deployments(model=model, llm_router=llm_router, valid_token=valid_token) + return True + + +def raise_when_key_reaches_only_other_teams_deployments( + model: str | Sequence[str], + llm_router: Router | None, + valid_token: UserAPIKeyAuth, +) -> None: + if llm_router is None or is_proxy_admin(valid_token): + return + requested_models: Final = (model,) if isinstance(model, str) else tuple(model) + foreign_model: Final = next( + (name for name in requested_models if llm_router.model_owned_by_other_teams(name, valid_token.team_id)), + None, + ) + if foreign_model is None: + return + raise ModelAccessDeniedProxyException( + message=model_access_denied_client_message(model=foreign_model), + internal_message=( + f"key not allowed to access model. Tried to access {foreign_model}, which is only deployed for other teams" + ), + type=ProxyErrorTypes.get_model_access_error_type_for_object(object_type="key"), + param="model", + code=status.HTTP_403_FORBIDDEN, + ) async def _personal_key_owner( @@ -5576,7 +5605,9 @@ async def can_key_call_resolved_model( skip_key_model_check: Final = valid_token.config or ( isinstance(valid_token.models, list) and SpecialModelNames.all_team_models.value in valid_token.models ) - if not skip_key_model_check: + if skip_key_model_check: + raise_when_key_reaches_only_other_teams_deployments(model=model, llm_router=llm_router, valid_token=valid_token) + else: await can_key_call_model( model=model, llm_model_list=llm_model_list, diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index c4c538083ee..58eeecaac67 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -5,6 +5,7 @@ import functools import itertools import re from collections.abc import Callable, Mapping, Sequence +from types import MappingProxyType from typing import Final import litellm @@ -61,7 +62,7 @@ def get_provider_models(provider: str, litellm_params: LiteLLM_Params | None = N def _get_models_from_access_groups( - model_access_groups: dict[str, list[str]], + model_access_groups: Mapping[str, Sequence[str]], all_models: list[str], include_model_access_groups: bool | None = False, proxy_model_list: Sequence[str] | None = None, @@ -108,8 +109,8 @@ async def get_mcp_server_ids( def get_key_models( user_api_key_dict: UserAPIKeyAuth, - proxy_model_list: list[str], - model_access_groups: dict[str, list[str]], + proxy_model_list: Sequence[str], + model_access_groups: Mapping[str, Sequence[str]], include_model_access_groups: bool | None = False, only_model_access_groups: bool | None = False, ) -> list[str]: @@ -152,8 +153,8 @@ def get_key_models( def get_team_models( team_models: list[str], - proxy_model_list: list[str], - model_access_groups: dict[str, list[str]], + proxy_model_list: Sequence[str], + model_access_groups: Mapping[str, Sequence[str]], include_model_access_groups: bool | None = False, ) -> list[str]: """ @@ -195,12 +196,12 @@ def get_team_models( def get_complete_model_list( key_models: Sequence[str], team_models: Sequence[str], - proxy_model_list: list[str], + proxy_model_list: Sequence[str], user_model: str | None, infer_model_from_keys: bool | None, return_wildcard_routes: bool | None = False, llm_router: Router | None = None, - model_access_groups: dict[str, list[str]] = {}, + model_access_groups: Mapping[str, Sequence[str]] = MappingProxyType({}), include_model_access_groups: bool | None = False, only_model_access_groups: bool | None = False, team_id: str | None = None, diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 99edfb940d7..33dc7c113c3 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -85,6 +85,7 @@ from litellm.proxy.auth.auth_checks import ( # noqa: F401 # legacy module expo jwt_key_mapping_cache_key, key_access_group_grants_model, key_model_aliases_for_auth_check, + raise_when_key_reaches_only_other_teams_deployments, resolve_and_validate_end_user_id, resolve_default_end_user_budget, route_skips_budget_checks, @@ -4534,49 +4535,50 @@ async def enforce_key_and_fallback_model_access( """ await _normalize_claude_model(request_data, valid_token, request, route) await _resolve_router_settings_model_group_alias(request_data, valid_token, request, route) - config: Final = valid_token.config + skip_key_model_check: Final = valid_token.config != {} or ( + isinstance(valid_token.models, list) and "all-team-models" in valid_token.models + ) + model: Final = _get_model_from_request_context( + request_data=request_data, + route=route, + request=request, + llm_router=llm_router, + team_id=valid_token.team_id, + ) + fallback_names: Final = tuple( + name + for target in iter_request_fallback_targets(request_data) + if (name := fallback_target_model_name(target)) is not None + ) - if config != {}: - model_list: Final = config.get("model_list", []) - new_model_list: Final = model_list - verbose_proxy_logger.debug("\n new llm router model list %s", new_model_list) - elif isinstance(valid_token.models, list) and "all-team-models" in valid_token.models: - pass - else: - model: Final = _get_model_from_request_context( - request_data=request_data, - route=route, - request=request, + if skip_key_model_check: + requested_models: Final = (model,) if isinstance(model, str) else tuple(model or ()) + for requested_name in dict.fromkeys((*requested_models, *fallback_names)): + raise_when_key_reaches_only_other_teams_deployments( + model=requested_name, llm_router=llm_router, valid_token=valid_token + ) + return + + if model is not None: + await can_key_call_model( + model=model, + llm_model_list=llm_model_list, + valid_token=valid_token, llm_router=llm_router, - team_id=valid_token.team_id, ) - if model is not None: - await can_key_call_model( - model=model, - llm_model_list=llm_model_list, - valid_token=valid_token, - llm_router=llm_router, - ) - - fallback_names: Final = tuple( - name - for target in iter_request_fallback_targets(request_data) - if (name := fallback_target_model_name(target)) is not None + for _name in dict.fromkeys(fallback_names): # dedupe, preserve order + await can_key_call_model( + model=_name, + llm_model_list=llm_model_list, + valid_token=valid_token, + llm_router=llm_router, + ) + await is_valid_fallback_model( + model=_name, + llm_router=llm_router, + user_model=None, ) - - for _name in dict.fromkeys(fallback_names): # dedupe, preserve order - await can_key_call_model( - model=_name, - llm_model_list=llm_model_list, - valid_token=valid_token, - llm_router=llm_router, - ) - await is_valid_fallback_model( - model=_name, - llm_router=llm_router, - user_model=None, - ) _enforce_key_and_fallback_model_access: Final = enforce_key_and_fallback_model_access # pyright: ignore[reportUnknownVariableType] # existing tests use this alias diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 64fb2791913..cc8a274cb04 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -15899,17 +15899,18 @@ def _resolve_model_grant_to_deployment_ids( time (see `_check_model_access_helper`), so both expand to every non-team deployment. A grant entry naming an access group also grants that group's members, and naming a deployed model that shares the name grants the model itself, matching the union the - call-time check applies. + call-time check applies. Team-owned deployments are never direct access, whatever the + grant names: they reach the caller through team membership (`access_via_team_ids`). """ if not models or SpecialModelNames.all_proxy_models.value in models: return tuple(llm_router.get_model_ids(exclude_team_models=True)) - access_groups: Final = llm_router.get_model_access_groups() + access_groups: Final = llm_router.get_model_access_groups_usable_by_team(None) granted_model_names: Final = tuple(name for model in models for name in (model, *access_groups.get(model, ()))) return tuple( model_id for name in granted_model_names - for deployment in (llm_router.get_model_list(model_name=name) or ()) + for deployment in llm_router.get_deployments_usable_by_team(name, None) if (model_id := deployment.get("model_info", {}).get("id", None)) is not None ) @@ -16646,7 +16647,9 @@ async def _gather_team_accessible_model_ids( ) -> set[str]: """Collect model IDs the team can use from router config and DB.""" team_accessible_model_ids: Final[set[str]] = set() - access_groups: Final = llm_router.get_model_access_groups() if llm_router else {} + access_groups: Final[Mapping[str, Sequence[str]]] = ( + llm_router.get_model_access_groups_usable_by_team(team_id) if llm_router else MappingProxyType({}) + ) if not team_object.models or SpecialModelNames.all_proxy_models.value in team_object.models: model_list: Final = llm_router.get_model_list() if llm_router else [] diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 0ddfc5ce62f..04160c2612e 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -162,6 +162,7 @@ from litellm.litellm_core_utils.duration_parser import get_budget_window_start from litellm.litellm_core_utils.env_utils import get_env_float, get_env_int from litellm.litellm_core_utils.litellm_logging import BilledRoundsDispatch, Logging, deployment_pricing_model_info from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token +from litellm.litellm_core_utils.resource_ownership import is_proxy_admin from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.safe_json_loads import safe_json_loads from litellm.litellm_core_utils.served_output_texts import ( @@ -9532,6 +9533,16 @@ async def _agent_access_group_visible_models( ) +def access_groups_visible_to_caller( + llm_router: "Router", + user_api_key_dict: "UserAPIKeyAuth", + team_id: str | None, +) -> Mapping[str, Sequence[str]]: + if team_id is None and is_proxy_admin(user_api_key_dict): + return llm_router.get_model_access_groups() + return llm_router.get_model_access_groups_usable_by_team(team_id) + + async def get_available_models_for_user( user_api_key_dict: "UserAPIKeyAuth", llm_router: Optional["Router"], @@ -9574,13 +9585,13 @@ async def get_available_models_for_user( get_team_models, ) - # Get proxy model list and access groups - if llm_router is None: - proxy_model_list = [] - model_access_groups = {} - else: - proxy_model_list = llm_router.get_model_names() - model_access_groups = llm_router.get_model_access_groups() + effective_team_id: Final = team_id or user_api_key_dict.team_id + proxy_model_list: Final[Sequence[str]] = llm_router.get_model_names() if llm_router is not None else () + model_access_groups: Final[Mapping[str, Sequence[str]]] = ( + access_groups_visible_to_caller(llm_router, user_api_key_dict, effective_team_id) + if llm_router is not None + else MappingProxyType({}) + ) requested_team_object: Final = ( await _get_validated_team_object( @@ -9614,8 +9625,6 @@ async def get_available_models_for_user( include_model_access_groups=include_model_access_groups, ) - effective_team_id: Final = team_id or user_api_key_dict.team_id - access_group_models: Final = ( await _get_access_group_models( user_api_key_dict=user_api_key_dict, diff --git a/litellm/router.py b/litellm/router.py index 5b18d36316f..6a490e17fb5 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -14055,6 +14055,37 @@ class Router: return access_groups + def _model_name_usable_by_team(self, model_name: str, team_id: str | None) -> bool: + return any( + self.deployment_usable_by_team(deployment, team_id) + for deployment in self.deployments_named(model_name) or () + ) + + def get_model_access_groups_usable_by_team(self, team_id: str | None) -> Mapping[str, Sequence[str]]: + return MappingProxyType( + { + group: usable_group_models + for group, model_names in self.get_model_access_groups().items() + if ( + usable_group_models := tuple( + name for name in model_names if self._model_name_usable_by_team(name, team_id) + ) + ) + } + ) + + def get_deployments_usable_by_team(self, model_name: str, team_id: str | None) -> tuple[DeploymentTypedDict, ...]: + return tuple( + deployment + for deployment in self.get_model_list(model_name=model_name, team_id=team_id) or () + if self.deployment_usable_by_team(deployment, team_id) + ) + + def model_owned_by_other_teams(self, model_name: str, team_id: str | None) -> bool: + if self.get_deployments_usable_by_team(model_name, team_id): + return False + return bool(self.get_model_list(model_name=model_name)) + def is_model_access_group_for_wildcard_route(self, model_access_group: str) -> bool: """ Return True if model access group is a wildcard route diff --git a/tests/unit/llms/anthropic/messages/test_safeguards.py b/tests/unit/llms/anthropic/messages/test_safeguards.py index 0537d0fbd38..203a4bf33da 100644 --- a/tests/unit/llms/anthropic/messages/test_safeguards.py +++ b/tests/unit/llms/anthropic/messages/test_safeguards.py @@ -345,6 +345,9 @@ class _Router: ) -> dict[str, list[str]]: return {} + def model_owned_by_other_teams(self, model_name: str, team_id: str | None) -> bool: + return False + async def acompletion(self, **kwargs: object) -> ModelResponse: self.calls.append(kwargs) return _classifier_response('{"verdicts": {"call_ls": {"flagged": false}}}') diff --git a/tests/unit/proxy/batches_endpoints/test_endpoints.py b/tests/unit/proxy/batches_endpoints/test_endpoints.py index 37d64a1fc79..2d975581571 100644 --- a/tests/unit/proxy/batches_endpoints/test_endpoints.py +++ b/tests/unit/proxy/batches_endpoints/test_endpoints.py @@ -194,6 +194,7 @@ def harness(monkeypatch: pytest.MonkeyPatch): router = MagicMock(spec=Router) router.model_group_alias = {} router.get_model_access_groups = MagicMock(return_value={}) + router.model_owned_by_other_teams = MagicMock(return_value=False) router.resolve_model_name_from_model_id = MagicMock(side_effect=lambda model_id: model_id) router.model_list = [] router.acreate_batch = AsyncMock(return_value=make_batch()) @@ -1453,6 +1454,7 @@ def retrieve_harness(): router = MagicMock(spec=Router) router.model_group_alias = {} router.get_model_access_groups = MagicMock(return_value={}) + router.model_owned_by_other_teams = MagicMock(return_value=False) router.resolve_model_name_from_model_id = MagicMock(side_effect=lambda model_id: model_id) router.model_list = [] router.aretrieve_batch = AsyncMock(return_value=make_batch()) @@ -2036,6 +2038,7 @@ def list_harness(): router = MagicMock(spec=Router) router.model_group_alias = {} router.get_model_access_groups = MagicMock(return_value={}) + router.model_owned_by_other_teams = MagicMock(return_value=False) router.resolve_model_name_from_model_id = MagicMock(side_effect=lambda model_id: model_id) router.model_list = [] router.alist_batches = AsyncMock(return_value=FakeListPage([])) @@ -2454,6 +2457,7 @@ def cancel_harness(): router = MagicMock(spec=Router) router.model_group_alias = {} router.get_model_access_groups = MagicMock(return_value={}) + router.model_owned_by_other_teams = MagicMock(return_value=False) router.resolve_model_name_from_model_id = MagicMock(side_effect=lambda model_id: model_id) router.model_list = [] router.acancel_batch = AsyncMock(return_value=make_batch())