diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index f173fbf2d18..c0c3d854068 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -284,6 +284,7 @@ class KeyManagementRoutes(str, enum.Enum): # team's `team_member_permissions`, non-admin members of that team may set # `access_group_ids` on keys they create/update. Default-deny. KEY_ACCESS_GROUP_ASSIGNMENT = "/key/access_group_assignment" + AUTO_ROUTER_MANAGE = "/auto_router/manage" # info and health routes KEY_INFO = "/key/info" @@ -651,6 +652,7 @@ class LiteLLMRoutes(enum.Enum): KeyManagementRoutes.KEY_RESET_SPEND.value, KeyManagementRoutes.KEY_ALIASES.value, KeyManagementRoutes.KEY_ACCESS_GROUP_ASSIGNMENT.value, + KeyManagementRoutes.AUTO_ROUTER_MANAGE.value, ] management_routes = ( diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index b256518b07b..10967690564 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -107,7 +107,7 @@ from litellm.proxy.utils import PrismaClient, ProxyLogging, log_db_metrics from litellm.repositories.budget_repository import BudgetRepository from litellm.repositories.object_permission_repository import ObjectPermissionRepository from litellm.repositories.organization_repository import OrganizationRepository -from litellm.repositories.prisma_protocols import RowT_co +from litellm.repositories.prisma_protocols import DatabaseClient, RowT_co from litellm.repositories.project_repository import ProjectRepository from litellm.repositories.table_repositories import ( AccessGroupRepository, @@ -4353,6 +4353,7 @@ async def can_key_call_model( llm_model_list: list | None, valid_token: UserAPIKeyAuth, llm_router: litellm.Router | None, + prisma_client: DatabaseClient | None = None, ) -> Literal[True]: """ Checks if token can call a given model @@ -4382,6 +4383,7 @@ async def can_key_call_model( if key_access_group_ids: models_from_groups: Final = await _get_models_from_access_groups( access_group_ids=key_access_group_ids, + prisma_client=prisma_client, ) if models_from_groups: return _can_object_call_model( @@ -4510,6 +4512,7 @@ async def can_team_access_model( team_object: LiteLLM_TeamTable | None, llm_router: Router | None, team_model_aliases: dict[str, str] | None = None, + prisma_client: DatabaseClient | None = None, ) -> Literal[True]: """ Returns True if the team can access a specific model. @@ -4532,6 +4535,7 @@ async def can_team_access_model( if team_access_group_ids: models_from_groups: Final = await _get_models_from_access_groups( access_group_ids=team_access_group_ids, + prisma_client=prisma_client, ) if models_from_groups: return _can_object_call_model( @@ -5211,6 +5215,8 @@ async def _check_team_member_model_access( prisma_client: Optional["PrismaClient"], user_api_key_cache: UserApiKeyCache, proxy_logging_obj: ProxyLogging, + team_membership: LiteLLM_TeamMembership | None = None, + team_membership_loaded: bool = False, ) -> None: """ Check if a team member's per-member model scope allows access to the requested model. @@ -5221,22 +5227,24 @@ async def _check_team_member_model_access( if valid_token.user_id is None or team_object.team_id is None: return - team_membership: Final = await get_team_membership( - user_id=valid_token.user_id, - team_id=team_object.team_id, - prisma_client=prisma_client, - user_api_key_cache=user_api_key_cache, - proxy_logging_obj=proxy_logging_obj, - ) + if not team_membership_loaded: + team_membership = await get_team_membership( + user_id=valid_token.user_id, + team_id=team_object.team_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + loaded_membership = team_membership if ( - team_membership is None - or team_membership.litellm_budget_table is None - or not team_membership.litellm_budget_table.allowed_models + loaded_membership is None + or loaded_membership.litellm_budget_table is None + or not loaded_membership.litellm_budget_table.allowed_models ): return # no per-member restriction — inherit team-level check - member_allowed_models: Final[list[str]] = team_membership.litellm_budget_table.allowed_models + member_allowed_models: Final[list[str]] = loaded_membership.litellm_budget_table.allowed_models try: _can_object_call_model( model=model, diff --git a/litellm/proxy/management_endpoints/auto_router_endpoints.py b/litellm/proxy/management_endpoints/auto_router_endpoints.py index 50716e5d474..200ed6c3bf3 100644 --- a/litellm/proxy/management_endpoints/auto_router_endpoints.py +++ b/litellm/proxy/management_endpoints/auto_router_endpoints.py @@ -40,6 +40,14 @@ from litellm.proxy.litellm_pre_call_utils import ( LiteLLMProxyRequestSetup, refresh_proxy_server_request_body_snapshot, ) +from litellm.proxy.management_endpoints.common_utils import ( + _is_user_team_admin, # pyright: ignore[reportPrivateUsage] # shared owner of team-admin membership +) +from litellm.proxy.management_helpers.auto_router_permissions import ( + authorize_member_auto_router_dependencies, + authorize_member_auto_router_team, + validate_member_auto_router_config, +) from litellm.repositories.autorouter_session_repository import AutoRouterSessionRepository from litellm.repositories.base_repository import SupportsModelDump from litellm.repositories.team_repository import TeamRepository @@ -72,13 +80,13 @@ from litellm.types.management_endpoints.auto_router_endpoints import ( ) if TYPE_CHECKING: - from fastapi import APIRouter, Depends, HTTPException, Query, status + from fastapi import APIRouter, Depends, HTTPException, Query, Request, status from litellm.proxy.utils import PrismaClient from litellm.router import Router else: try: - from fastapi import APIRouter, Depends, HTTPException, Query, status + from fastapi import APIRouter, Depends, HTTPException, Query, Request, status except ImportError: # fastapi is only required for proxy, not for SDK usage pass @@ -201,21 +209,14 @@ async def _query_raw(prisma_client: "PrismaClient", query: str, *args: object) - return await prisma_client.db.query_raw(query, *args) -async def _authorize_router_dry_run(user_api_key_dict: UserAPIKeyAuth, team_id: str | None) -> None: - """Allow exactly the callers who could create this router. - - Both dry runs are gated like the write they rehearse rather than as reads: a proxy - admin, or a team admin naming their own team, matching /model/new. Routing a test - prompt can also spend money (an `llm` classifier config calls its classifier, a - semantic config embeds the prompt), so a read-level gate would be too loose anyway. - """ +async def _authorize_router_dry_run(user_api_key_dict: UserAPIKeyAuth, team_id: str | None) -> LiteLLM_TeamTable | None: from litellm.proxy.management_endpoints.model_management_endpoints import ( ModelManagementAuthChecks, ) from litellm.proxy.proxy_server import premium_user, prisma_client if user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN: - return + return None if team_id is None: raise HTTPException( @@ -244,12 +245,47 @@ async def _authorize_router_dry_run(user_api_key_dict: UserAPIKeyAuth, team_id: }, ) - ModelManagementAuthChecks.can_user_make_team_model_call( - team_id=team_id, + team: Final = LiteLLM_TeamTable.model_validate(team_row.model_dump()) + if _is_user_team_admin(user_api_key_dict=user_api_key_dict, team_obj=team): + ModelManagementAuthChecks.can_user_make_team_model_call( + team_id=team_id, + user_api_key_dict=user_api_key_dict, + team_obj=team, + premium_user=premium_user, + ) + return None + authorize_member_auto_router_team( user_api_key_dict=user_api_key_dict, - team_obj=LiteLLM_TeamTable.model_validate(team_row.model_dump()), + team=team, premium_user=premium_user, ) + return team + + +async def _authorize_member_dry_run_config( + *, + config: Mapping[str, object], + default_model: str | None, + user_api_key_dict: UserAPIKeyAuth, + team: LiteLLM_TeamTable, +) -> UserAPIKeyAuth: + from litellm.proxy.proxy_server import llm_router, prisma_client + + if prisma_client is None or llm_router is None: + raise HTTPException(status_code=503, detail="Cannot verify auto-router model access") + validated: Final = validate_member_auto_router_config(config) + scoped_actor: Final = user_api_key_dict.model_copy( + update=MappingProxyType({"team_id": team.team_id, "team_models": team.models, "org_id": team.organization_id}) + ) + await authorize_member_auto_router_dependencies( + config=validated, + default_model=default_model, + user_api_key_dict=scoped_actor, + team=team, + prisma_client=prisma_client, + llm_router=llm_router, + ) + return scoped_actor def _models_this_test_can_call(config: RequestComplexityRouterConfig) -> tuple[str, ...]: @@ -326,16 +362,23 @@ async def validate_complexity_router_config( Runs the same check every write path runs (the router's own pydantic model), so a form can show the backend's exact verdict while the operator is still editing rather than after a - rejected save. Gated exactly like the save it rehearses: a proxy admin, or a team admin - naming their own team. Nothing is created, routed, or billed. + rejected save. Uses the same team opt-in and model-access checks as configuration + writes for members. Nothing is created, routed, or billed. """ - await _authorize_router_dry_run(user_api_key_dict=user_api_key_dict, team_id=data.team_id) + member_team: Final = await _authorize_router_dry_run(user_api_key_dict=user_api_key_dict, team_id=data.team_id) from litellm.router_utils.auto_router_model_naming import ( validate_complexity_router_config_write, ) error: Final = validate_complexity_router_config_write(data.complexity_router_config) + if error is None and member_team is not None: + await _authorize_member_dry_run_config( + config=data.complexity_router_config, + default_model=None, + user_api_key_dict=user_api_key_dict, + team=member_team, + ) return ComplexityRouterConfigValidationResponse(valid=error is None, error=error) @@ -349,6 +392,7 @@ async def validate_complexity_router_config( async def preview_auto_router_routing( data: AutoRouterRoutingTestRequest, user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], + http_request: Request, ) -> AutoRouterRoutingTestResponse: """ Route a single request through a complexity-router config and report where it landed. @@ -392,7 +436,34 @@ async def preview_auto_router_routing( ) from litellm.proxy.utils import get_available_models_for_user - await _authorize_router_dry_run(user_api_key_dict=user_api_key_dict, team_id=data.team_id) + member_team: Final = await _authorize_router_dry_run(user_api_key_dict=user_api_key_dict, team_id=data.team_id) + actor: Final = ( + await _authorize_member_dry_run_config( + config=data.complexity_router_config.model_dump(exclude_none=True), + default_model=data.default_model, + user_api_key_dict=user_api_key_dict, + team=member_team, + ) + if member_team is not None + else user_api_key_dict + ) + request_data: Final[dict[str, object]] = { # mutable-ok: auth and routing enrich this request in place + **data.wire_body(), + "metadata": {}, # mutable-ok: centralized auth and identity stamping share this metadata bucket + "proxy_server_request": {"body": None}, # mutable-ok: the snapshot owner fills this body in place + } + + if member_team is not None and _models_this_test_can_call(data.complexity_router_config): + from litellm.proxy.auth.user_api_key_auth import ( + _run_centralized_common_checks, # pyright: ignore[reportPrivateUsage] # reuse the serving admission policy + ) + + await _run_centralized_common_checks( + user_api_key_auth_obj=actor, + request=http_request, + request_data=request_data, + route="/auto_router/test_routing", + ) if llm_router is None: raise HTTPException( @@ -404,7 +475,7 @@ async def preview_auto_router_routing( await _authorize_models_this_test_can_call( config=data.complexity_router_config, - user_api_key_dict=user_api_key_dict, + user_api_key_dict=actor, llm_router=llm_router, ) @@ -417,12 +488,8 @@ async def preview_auto_router_routing( ) request_kwargs: Final = LiteLLMProxyRequestSetup.add_user_api_key_auth_to_request_metadata( - data={ # mutable-ok: the request-metadata helper takes and returns request kwargs as a dict - **data.wire_body(), - "metadata": {}, # mutable-ok: the request-metadata helper writes the auth fields into this dict - "proxy_server_request": {"body": None}, # mutable-ok: the snapshot owner fills body in place - }, - user_api_key_dict=user_api_key_dict, + data=request_data, + user_api_key_dict=actor, _metadata_variable_name="metadata", ) refresh_proxy_server_request_body_snapshot(request_kwargs) diff --git a/litellm/repositories/prisma_protocols.py b/litellm/repositories/prisma_protocols.py index d962934dfb1..93b8c5c7cd7 100644 --- a/litellm/repositories/prisma_protocols.py +++ b/litellm/repositories/prisma_protocols.py @@ -12,6 +12,11 @@ from typing import Protocol, TypeVar RowT_co = TypeVar("RowT_co", covariant=True) +class DatabaseClient(Protocol): + @property + def db(self) -> object: ... + + class TableActions(Protocol[RowT_co]): """The prisma-client-py per-model action surface, keyed to the row it returns. diff --git a/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py index ef843adad98..2735dde71c8 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py @@ -7,7 +7,7 @@ from pathlib import Path from typing import Final import pytest -from fastapi import HTTPException +from fastapi import HTTPException, Request from pydantic import ValidationError from litellm.proxy._types import ( @@ -26,6 +26,10 @@ from litellm.types.management_endpoints.auto_router_endpoints import ( ) from litellm.types.utils import Choices, Message, ModelResponse +ROUTING_HTTP_REQUEST: Final = Request( + {"type": "http", "method": "POST", "path": "/auto_router/test_routing", "headers": []} +) + ADMIN = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-test", user_id="admin") @@ -94,6 +98,7 @@ async def _route_body(body: Mapping[str, object], monkeypatch: pytest.MonkeyPatc monkeypatch.setattr(proxy_server, "llm_router", _router()) return await preview_auto_router_routing( + http_request=ROUTING_HTTP_REQUEST, data=_request_from(body, **config_overrides), user_api_key_dict=ADMIN, ) @@ -121,6 +126,7 @@ async def _classifier_user_payload(body: Mapping[str, object], monkeypatch: pyte monkeypatch.setattr(proxy_server, "llm_router", router) await preview_auto_router_routing( + http_request=ROUTING_HTTP_REQUEST, data=_request_from(body, classifier_type="llm", classifier_llm_config={"model": "classifier-model"}), user_api_key_dict=ADMIN, ) @@ -198,6 +204,7 @@ async def test_llm_classifier_call_is_billed_to_the_calling_key(monkeypatch: pyt monkeypatch.setattr(proxy_server, "llm_router", router) response = await preview_auto_router_routing( + http_request=ROUTING_HTTP_REQUEST, data=_request( "what is 2+2", classifier_type="llm", @@ -359,6 +366,7 @@ async def test_a_key_that_cannot_call_the_classifier_model_is_rejected_before_it with pytest.raises(ProxyException) as exc_info: await preview_auto_router_routing( + http_request=ROUTING_HTTP_REQUEST, data=_request("what is 2+2", **config_overrides), user_api_key_dict=UserAPIKeyAuth( user_role=LitellmUserRoles.PROXY_ADMIN, @@ -388,6 +396,7 @@ async def test_a_key_over_its_budget_cannot_run_a_classifier_config(monkeypatch: with pytest.raises(ProxyException) as exc_info: await preview_auto_router_routing( + http_request=ROUTING_HTTP_REQUEST, data=_request( "what is 2+2", classifier_type="llm", @@ -413,6 +422,7 @@ async def test_a_heuristic_config_does_not_need_a_budget(monkeypatch: pytest.Mon monkeypatch.setattr(proxy_server, "llm_router", _router()) response = await preview_auto_router_routing( + http_request=ROUTING_HTTP_REQUEST, data=_request("what is 2+2"), user_api_key_dict=UserAPIKeyAuth( user_role=LitellmUserRoles.PROXY_ADMIN, @@ -434,7 +444,9 @@ async def test_no_llm_router_on_the_proxy_is_a_500(monkeypatch: pytest.MonkeyPat monkeypatch.setattr(proxy_server, "llm_router", None) with pytest.raises(HTTPException) as exc_info: - await preview_auto_router_routing(data=_request("what is 2+2"), user_api_key_dict=ADMIN) + await preview_auto_router_routing( + http_request=ROUTING_HTTP_REQUEST, data=_request("what is 2+2"), user_api_key_dict=ADMIN + ) assert exc_info.value.status_code == 500 @@ -447,6 +459,7 @@ async def test_non_admin_without_a_team_is_rejected(monkeypatch: pytest.MonkeyPa with pytest.raises(HTTPException) as exc_info: await preview_auto_router_routing( + http_request=ROUTING_HTTP_REQUEST, data=_request("what is 2+2"), user_api_key_dict=UserAPIKeyAuth( user_role=LitellmUserRoles.INTERNAL_USER, api_key="sk-user", user_id="user" @@ -2712,12 +2725,16 @@ async def test_routing_test_never_confirms_models_the_caller_cannot_use(monkeypa ) monkeypatch.setattr(proxy_server, "prisma_client", _team_prisma("team-probe", models=["mid-model"])) - probing = await preview_auto_router_routing(data=_request("team-probe"), user_api_key_dict=team_admin) + probing = await preview_auto_router_routing( + http_request=ROUTING_HTTP_REQUEST, data=_request("team-probe"), user_api_key_dict=team_admin + ) assert probing.routed_model == "cheap-model" assert probing.routed_model_configured is False monkeypatch.setattr(proxy_server, "prisma_client", _team_prisma("team-grant", models=["cheap-model"])) - granted = await preview_auto_router_routing(data=_request("team-grant"), user_api_key_dict=team_admin) + granted = await preview_auto_router_routing( + http_request=ROUTING_HTTP_REQUEST, data=_request("team-grant"), user_api_key_dict=team_admin + ) assert granted.routed_model == "cheap-model" assert granted.routed_model_configured is True