mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
Merge 258dd03f11 into e16d79d635
This commit is contained in:
commit
cbac20b1b1
5 changed files with 289 additions and 11 deletions
|
|
@ -6,6 +6,8 @@ from typing import TYPE_CHECKING, Final
|
|||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_from_kwargs
|
||||
from litellm.router_utils.auto_router_model_naming import is_complexity_router_model
|
||||
from litellm.types.management_endpoints.auto_router_endpoints import RequestComplexityRouterConfig
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.router import Router
|
||||
|
|
@ -29,7 +31,16 @@ async def authorize_member_auto_router_inference(
|
|||
if deployment is None:
|
||||
return
|
||||
model_info: Final = _mapping(deployment.get("model_info"))
|
||||
if model_info is None or model_info.get("member_auto_router") is not True:
|
||||
params: Final = _mapping(deployment.get("litellm_params"))
|
||||
member_router: Final = model_info is not None and model_info.get("member_auto_router") is True
|
||||
team_router: Final = (
|
||||
model_info is not None
|
||||
and bool(model_info.get("team_id"))
|
||||
and params is not None
|
||||
and isinstance(model := params.get("model"), str)
|
||||
and is_complexity_router_model(model)
|
||||
)
|
||||
if model_info is None or not (member_router or team_router):
|
||||
return
|
||||
|
||||
from fastapi import HTTPException
|
||||
|
|
@ -51,6 +62,8 @@ async def authorize_member_auto_router_inference(
|
|||
|
||||
metadata: Final = _mapping(request_kwargs.get(get_metadata_variable_name_from_kwargs(request_kwargs)))
|
||||
actor: Final = metadata.get("user_api_key_auth") if metadata is not None else None
|
||||
if not member_router and isinstance(actor, UserAPIKeyAuth) and actor.user_role == LitellmUserRoles.PROXY_ADMIN:
|
||||
return
|
||||
team_id: Final = model_info.get("team_id")
|
||||
if not isinstance(actor, UserAPIKeyAuth) or not isinstance(team_id, str) or not team_id:
|
||||
raise HTTPException(status_code=403, detail="Member auto-routers require authenticated team access")
|
||||
|
|
@ -79,14 +92,17 @@ async def authorize_member_auto_router_inference(
|
|||
raise HTTPException(status_code=403, detail="You are no longer a member of this auto-router's team")
|
||||
if team.blocked:
|
||||
raise HTTPException(status_code=403, detail="This auto router's team is blocked.")
|
||||
params: Final = _mapping(deployment.get("litellm_params"))
|
||||
if params is None:
|
||||
raise HTTPException(status_code=403, detail="The member auto-router configuration is invalid")
|
||||
raw_config: Final = _mapping(params.get("complexity_router_config"))
|
||||
if raw_config is None:
|
||||
raise HTTPException(status_code=403, detail="The member auto-router configuration is invalid")
|
||||
default_model: Final = params.get("complexity_router_default_model")
|
||||
config: Final = validate_member_auto_router_config(raw_config)
|
||||
config: Final = (
|
||||
validate_member_auto_router_config(raw_config)
|
||||
if member_router
|
||||
else RequestComplexityRouterConfig.model_validate(raw_config)
|
||||
)
|
||||
membership: Final = (
|
||||
await get_team_membership(
|
||||
user_id=actor.user_id,
|
||||
|
|
@ -128,7 +144,7 @@ async def authorize_member_auto_router_inference(
|
|||
default_model=default_model if isinstance(default_model, str) else None,
|
||||
user_api_key_dict=actor,
|
||||
team=team,
|
||||
prisma_client=None,
|
||||
prisma_client=prisma_client,
|
||||
llm_router=llm_router,
|
||||
dependency_objects=MemberAutoRouterDependencyObjects(
|
||||
membership=membership, organization=organization, project=project
|
||||
|
|
|
|||
|
|
@ -143,6 +143,7 @@ from litellm.router_utils.auto_router_model_naming import (
|
|||
from litellm.router_utils.auto_router_tuning_baseline import is_mutable_tuned_candidate, tuning_quota_violation
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
from litellm.types.llms.bedrock import AwsSessionTag
|
||||
from litellm.types.management_endpoints.auto_router_endpoints import RequestComplexityRouterConfig
|
||||
from litellm.types.proxy.management_endpoints.model_management_endpoints import (
|
||||
AutoRouterClassifierDefaultPromptResponse,
|
||||
PatchModelRequest,
|
||||
|
|
@ -295,7 +296,9 @@ class _TeamRow(Protocol):
|
|||
|
||||
|
||||
class _TeamLookupTable(Protocol):
|
||||
def find_unique(self, *, where: Mapping[str, object]) -> Awaitable[_TeamRow | None]: ...
|
||||
def find_unique(
|
||||
self, *, where: Mapping[str, object], include: Mapping[str, bool] | None = None
|
||||
) -> Awaitable[_TeamRow | None]: ...
|
||||
|
||||
async def find_many(self, *, where: Mapping[str, object]) -> Sequence[_TeamRow]: ...
|
||||
|
||||
|
|
@ -2215,6 +2218,61 @@ def _canonical_session_tags(tags: Sequence[AwsSessionTag]) -> tuple[tuple[str, s
|
|||
return tuple(sorted((tag["Key"], tag["Value"]) for tag in tags))
|
||||
|
||||
|
||||
async def _authorize_team_router_dependencies_on_write(
|
||||
*,
|
||||
incoming: Deployment | updateDeployment | None,
|
||||
existing: Deployment | None,
|
||||
actor: UserAPIKeyAuth,
|
||||
team: LiteLLM_TeamTable,
|
||||
prisma_client: PrismaClient,
|
||||
) -> None:
|
||||
from litellm.proxy.proxy_server import llm_router
|
||||
|
||||
incoming_params: Final = incoming.litellm_params if incoming is not None else None
|
||||
existing_params: Final = existing.litellm_params if existing is not None else None
|
||||
if not is_complexity_router_model(_effective_model(incoming_params, existing_params)):
|
||||
return
|
||||
if llm_router is None:
|
||||
raise HTTPException(status_code=400, detail="An auto-router model catalog is required.")
|
||||
requested_team_id: Final = incoming.model_info.team_id if incoming is not None and incoming.model_info else None
|
||||
destination_row: Final = (
|
||||
await _repo_team_table(prisma_client).find_unique(
|
||||
where={"team_id": requested_team_id}, include={"litellm_model_table": True}
|
||||
)
|
||||
if requested_team_id is not None and requested_team_id != team.team_id
|
||||
else None
|
||||
)
|
||||
if requested_team_id is not None and requested_team_id != team.team_id and destination_row is None:
|
||||
raise HTTPException(status_code=400, detail="The destination team does not exist.")
|
||||
destination: Final = LiteLLM_TeamTable.model_validate(destination_row.model_dump()) if destination_row else team
|
||||
ModelManagementAuthChecks.can_user_make_team_model_call(
|
||||
team_id=destination.team_id, user_api_key_dict=actor, team_obj=destination, premium_user=True
|
||||
)
|
||||
try:
|
||||
config: Final = RequestComplexityRouterConfig.model_validate(
|
||||
_effective_complexity_router_config(incoming_params, existing_params)
|
||||
)
|
||||
except ValidationError as error:
|
||||
raise HTTPException(status_code=400, detail="The auto-router configuration is invalid.") from error
|
||||
supplied_default: Final = incoming_params.complexity_router_default_model if incoming_params is not None else None
|
||||
stored_default: Final = existing_params.complexity_router_default_model if existing_params is not None else None
|
||||
default_model: Final = (
|
||||
supplied_default
|
||||
if supplied_default is not None
|
||||
else decrypt_value_helper(stored_default, key="complexity_router_default_model", return_original_value=True)
|
||||
if stored_default is not None
|
||||
else None
|
||||
)
|
||||
await authorize_member_auto_router_dependencies(
|
||||
config=config,
|
||||
default_model=default_model,
|
||||
user_api_key_dict=actor,
|
||||
team=destination,
|
||||
prisma_client=prisma_client,
|
||||
llm_router=llm_router,
|
||||
)
|
||||
|
||||
|
||||
class ModelManagementAuthChecks:
|
||||
"""
|
||||
Common auth checks for model management endpoints
|
||||
|
|
@ -2429,7 +2487,8 @@ class ModelManagementAuthChecks:
|
|||
## Check team model auth
|
||||
if model_params.model_info.team_id is not None:
|
||||
team_obj_row: Final = await _repo_team_table(prisma_client).find_unique(
|
||||
where={"team_id": model_params.model_info.team_id}
|
||||
where={"team_id": model_params.model_info.team_id},
|
||||
include={"litellm_model_table": True},
|
||||
)
|
||||
if team_obj_row is None:
|
||||
# The team was deleted. Callers that opt in (e.g. model deletion) may
|
||||
|
|
@ -2466,12 +2525,21 @@ class ModelManagementAuthChecks:
|
|||
llm_router=llm_router,
|
||||
)
|
||||
|
||||
return ModelManagementAuthChecks.can_user_make_team_model_call(
|
||||
ModelManagementAuthChecks.can_user_make_team_model_call(
|
||||
team_id=model_params.model_info.team_id,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
team_obj=team_obj,
|
||||
premium_user=premium_user,
|
||||
)
|
||||
if member_operation is not None and user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN:
|
||||
await _authorize_team_router_dependencies_on_write(
|
||||
incoming=model_params if member_operation == "create" else incoming_model_params,
|
||||
existing=model_params if member_operation == "update" else None,
|
||||
actor=user_api_key_dict,
|
||||
team=team_obj,
|
||||
prisma_client=prisma_client,
|
||||
)
|
||||
return True
|
||||
## Check non-team model auth
|
||||
elif user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN:
|
||||
raise HTTPException(
|
||||
|
|
|
|||
|
|
@ -195,7 +195,17 @@ async def authorize_member_auto_router_dependencies(
|
|||
(
|
||||
dependency,
|
||||
dependency.model_name,
|
||||
llm_router.get_model_list(model_name=dependency.model_name, team_id=team.team_id),
|
||||
tuple(
|
||||
deployment
|
||||
for deployment in llm_router.get_model_list(
|
||||
model_name=aliases.get(dependency.model_name, dependency.model_name)
|
||||
if aliases
|
||||
else dependency.model_name,
|
||||
team_id=team.team_id,
|
||||
)
|
||||
or ()
|
||||
if Router._deployment_usable_by_team(deployment, team.team_id) # pyright: ignore[reportPrivateUsage] # use the serving path's team boundary
|
||||
),
|
||||
)
|
||||
for dependency in dependencies
|
||||
):
|
||||
|
|
|
|||
|
|
@ -329,7 +329,7 @@ class MockPrismaClient:
|
|||
self.sibling_deployments = sibling_deployments or []
|
||||
self.db = self
|
||||
|
||||
async def find_unique(self, where):
|
||||
async def find_unique(self, where, include=None):
|
||||
if self.team_exists:
|
||||
return LiteLLM_TeamTable(
|
||||
team_id=where["team_id"],
|
||||
|
|
@ -7774,10 +7774,10 @@ class TestTeamMemberAutoRouterWrites:
|
|||
monkeypatch.setenv("LITELLM_SALT_KEY", "member-router-test-salt")
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _environment(self, database: MagicMock, row: LiteLLM_ProxyModelTable) -> Iterator[None]:
|
||||
def _environment(self, database: MagicMock, row: LiteLLM_ProxyModelTable, catalog: Router | None = None) -> Iterator[None]:
|
||||
with (
|
||||
patch("litellm.proxy.proxy_server.prisma_client", database), # test-quality-ok: [TQ008] endpoint storage singleton injection
|
||||
patch("litellm.proxy.proxy_server.llm_router", self._catalog()), # test-quality-ok: [TQ008] inject real destination model catalog
|
||||
patch("litellm.proxy.proxy_server.llm_router", catalog or self._catalog()), # test-quality-ok: [TQ008] inject real destination model catalog
|
||||
patch("litellm.proxy.proxy_server.store_model_in_db", True), # test-quality-ok: [TQ008] endpoint storage mode singleton
|
||||
patch("litellm.proxy.proxy_server.premium_user", True), # test-quality-ok: [TQ008] inject licensed process state
|
||||
patch("litellm.proxy.proxy_server._license_check.auto_router_capability_limit", return_value=None), # test-quality-ok: [TQ008] inject unlimited license result
|
||||
|
|
@ -7852,6 +7852,102 @@ class TestTeamMemberAutoRouterWrites:
|
|||
"model_info": {"id": "allowed-id"},
|
||||
}])
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("endpoint", ["create", "patch", "legacy"])
|
||||
@pytest.mark.parametrize("dependency", ["tier", "default", "classifier", "inherited-default"])
|
||||
async def test_team_admin_dependencies_are_bounded_before_persistence(
|
||||
self, endpoint: str, dependency: str
|
||||
) -> None:
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import add_new_model
|
||||
|
||||
original: Final = self._row()
|
||||
row: Final = original.model_copy(update={"litellm_params": {
|
||||
**original.litellm_params,
|
||||
"complexity_router_default_model": "outside" if dependency == "inherited-default" else "allowed",
|
||||
}})
|
||||
team: Final = self._team(enabled=False).model_copy(update={
|
||||
"members_with_roles": [Member(user_id="admin", role="admin")],
|
||||
})
|
||||
database: Final = self._database(team, row)
|
||||
actor: Final = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.INTERNAL_USER, team_id=None)
|
||||
config: Final = {
|
||||
"classifier_type": "llm" if dependency == "classifier" else "heuristic",
|
||||
"tiers": {"SIMPLE": "outside" if dependency == "tier" else "allowed"},
|
||||
**({"classifier_llm_config": {"model": "outside"}} if dependency == "classifier" else {}),
|
||||
}
|
||||
params: Final = {
|
||||
"complexity_router_config": config,
|
||||
**({"complexity_router_default_model": "outside"} if dependency == "default" else {}),
|
||||
}
|
||||
request: Final = updateDeployment(
|
||||
litellm_params=updateLiteLLMParams.model_validate(params), model_info=ModelInfo(id=row.model_id)
|
||||
)
|
||||
deployment: Final = Deployment(
|
||||
model_name="new-admin-router",
|
||||
litellm_params=LiteLLM_Params.model_validate({
|
||||
"model": "auto_router/complexity_router", **params,
|
||||
**({"complexity_router_default_model": "outside"} if dependency == "inherited-default" else {}),
|
||||
}),
|
||||
model_info=ModelInfo(id=row.model_id, team_id=team.team_id),
|
||||
)
|
||||
catalog: Final = Router(model_list=[
|
||||
{"model_name": name, "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "fake"}}
|
||||
for name in ("allowed", "outside")
|
||||
])
|
||||
operation: Final = (
|
||||
add_new_model(deployment, actor) if endpoint == "create"
|
||||
else patch_model(row.model_id, request, actor) if endpoint == "patch"
|
||||
else update_model(request, actor)
|
||||
)
|
||||
with self._environment(database, row, catalog):
|
||||
with pytest.raises(ProxyException) as denied:
|
||||
await operation
|
||||
assert denied.value.code == "403"
|
||||
database.db.litellm_proxymodeltable.create.assert_not_awaited()
|
||||
database.db.litellm_proxymodeltable.update.assert_not_awaited()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("endpoint", ["patch", "legacy"])
|
||||
async def test_team_admin_can_update_another_users_router_without_member_opt_in(self, endpoint: str) -> None:
|
||||
row: Final = self._row()
|
||||
team: Final = self._team(enabled=False).model_copy(update={
|
||||
"members_with_roles": [Member(user_id="admin", role="admin")],
|
||||
})
|
||||
database: Final = self._database(team, row)
|
||||
actor: Final = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.INTERNAL_USER)
|
||||
request: Final = updateDeployment(
|
||||
litellm_params=updateLiteLLMParams(complexity_router_config={"tiers": {"SIMPLE": "allowed"}}),
|
||||
model_info=ModelInfo(id=row.model_id),
|
||||
)
|
||||
with self._environment(database, row):
|
||||
if endpoint == "patch":
|
||||
await patch_model(row.model_id, request, actor)
|
||||
else:
|
||||
await update_model(request, actor)
|
||||
saved: Final = database.db.litellm_proxymodeltable.update.await_args.kwargs["data"]
|
||||
assert json.loads(saved["litellm_params"])["complexity_router_config"]["tiers"] == {"SIMPLE": "allowed"}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("endpoint", ["patch", "legacy"])
|
||||
async def test_team_admin_router_transfer_checks_destination_dependencies(self, endpoint: str) -> None:
|
||||
row: Final = self._row()
|
||||
source: Final = self._team(enabled=False).model_copy(update={
|
||||
"members_with_roles": [Member(user_id="admin", role="admin")],
|
||||
})
|
||||
destination: Final = source.model_copy(update={"team_id": "destination", "models": ["other"]})
|
||||
database: Final = self._database(source, row)
|
||||
database.db.litellm_teamtable.find_unique.side_effect = [source, destination]
|
||||
actor: Final = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.INTERNAL_USER)
|
||||
request: Final = updateDeployment(
|
||||
litellm_params=updateLiteLLMParams(complexity_router_config={"tiers": {"SIMPLE": "allowed"}}),
|
||||
model_info=ModelInfo(id=row.model_id, team_id="destination"),
|
||||
)
|
||||
operation: Final = patch_model(row.model_id, request, actor) if endpoint == "patch" else update_model(request, actor)
|
||||
with self._environment(database, row), pytest.raises(ProxyException) as denied:
|
||||
await operation
|
||||
assert denied.value.code == "403"
|
||||
database.db.litellm_proxymodeltable.update.assert_not_awaited()
|
||||
|
||||
@staticmethod
|
||||
def _classifier_config(classifier: Mapping[str, object], legacy: bool) -> Mapping[str, object]:
|
||||
return {
|
||||
|
|
|
|||
|
|
@ -28,6 +28,94 @@ from litellm.router import Router
|
|||
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo, updateDeployment
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("bucket", ["metadata", "litellm_metadata"])
|
||||
@pytest.mark.parametrize("member_marker", [False, None])
|
||||
async def test_unmarked_team_router_checks_dependencies_on_inference(
|
||||
catalog: Router, monkeypatch: pytest.MonkeyPatch, bucket: str, member_marker: bool | None
|
||||
) -> None:
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy.auth.auto_router_checks import authorize_member_auto_router_inference
|
||||
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
|
||||
from litellm.proxy._types import LiteLLM_TeamTableCachedObj
|
||||
|
||||
cache: Final = UserApiKeyCache()
|
||||
team: Final = LiteLLM_TeamTableCachedObj.model_validate(_team().model_dump())
|
||||
await cache.async_set_cache(key="team_id:team-a", value=team, model_type=LiteLLM_TeamTableCachedObj)
|
||||
monkeypatch.setattr(proxy_server, "user_api_key_cache", cache)
|
||||
monkeypatch.setattr(proxy_server, "prisma_client", _Client())
|
||||
actor: Final = _actor(user_id=None, team_id="team-a", models=[])
|
||||
with pytest.raises(ProxyException) as denied:
|
||||
await authorize_member_auto_router_inference(
|
||||
deployment={
|
||||
"model_info": {"team_id": "team-a", "member_auto_router": member_marker},
|
||||
"litellm_params": {"model": "auto_router/complexity_router", "complexity_router_config": {
|
||||
"classifier_type": "heuristic", "tiers": {"SIMPLE": "other"},
|
||||
}},
|
||||
},
|
||||
request_kwargs={bucket: {"user_api_key_auth": actor}},
|
||||
llm_router=catalog,
|
||||
)
|
||||
assert denied.value.code == "403"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("shape", ["global", "proxy-admin", "ordinary"])
|
||||
async def test_explicit_admin_and_global_router_delegation_remain_unchanged(catalog: Router, shape: str) -> None:
|
||||
from litellm.proxy.auth.auto_router_checks import authorize_member_auto_router_inference
|
||||
|
||||
result: Final = await authorize_member_auto_router_inference(
|
||||
deployment={
|
||||
"model_info": {} if shape == "global" else {"team_id": "team-a"},
|
||||
"litellm_params": {"model": "openai/gpt-4o-mini" if shape == "ordinary" else "auto_router/complexity_router"},
|
||||
},
|
||||
request_kwargs={"metadata": {"user_api_key_auth": _actor(user_role=LitellmUserRoles.PROXY_ADMIN)}},
|
||||
llm_router=catalog,
|
||||
)
|
||||
assert result is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("owner", ["team-a", "team-b"])
|
||||
async def test_dependency_catalog_respects_team_ownership(owner: str) -> None:
|
||||
catalog: Final = Router(model_list=[{
|
||||
"model_name": "allowed", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "fake"},
|
||||
"model_info": {"team_id": owner},
|
||||
}])
|
||||
operation: Final = authorize_member_auto_router_dependencies(
|
||||
config=validate_member_auto_router_config({"tiers": {"SIMPLE": "allowed"}}), default_model=None,
|
||||
user_api_key_dict=_actor(), team=_team(), prisma_client=_Client(), llm_router=catalog,
|
||||
)
|
||||
if owner == "team-b":
|
||||
with pytest.raises(HTTPException) as denied:
|
||||
await operation
|
||||
assert denied.value.status_code == 400
|
||||
else:
|
||||
assert await operation is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("grant", ["alias", "wildcard", "model-group", "access-group"])
|
||||
async def test_dependency_access_uses_existing_alias_and_grant_semantics(grant: str) -> None:
|
||||
catalog: Final = Router(
|
||||
model_list=[{
|
||||
"model_name": "allowed", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "fake"},
|
||||
"model_info": {"access_groups": ["permitted-group"]},
|
||||
}],
|
||||
model_group_alias={"public": "allowed"} if grant == "model-group" else {},
|
||||
)
|
||||
target: Final = "public" if grant in ("alias", "model-group") else "allowed"
|
||||
team: Final = _team(
|
||||
models=["allow*"] if grant == "wildcard" else ["permitted-group"] if grant == "access-group" else ["allowed"],
|
||||
litellm_model_table={"model_aliases": {"public": "allowed"}, "created_by": "admin", "updated_by": "admin"} if grant == "alias" else None,
|
||||
)
|
||||
result: Final = await authorize_member_auto_router_dependencies(
|
||||
config=validate_member_auto_router_config({"tiers": {"SIMPLE": target}}), default_model=None,
|
||||
user_api_key_dict=_actor(models=[]), team=team, prisma_client=_Client(), llm_router=catalog,
|
||||
)
|
||||
assert result is None
|
||||
|
||||
|
||||
class _ReadTable:
|
||||
async def find_unique(self, where: Mapping[str, object], include: Mapping[str, object] | None = None) -> None:
|
||||
return None
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue