diff --git a/litellm/proxy/management_endpoints/usage_endpoints/ai_usage_chat.py b/litellm/proxy/management_endpoints/usage_endpoints/ai_usage_chat.py index bc4ff986f27..fa1f22660b1 100644 --- a/litellm/proxy/management_endpoints/usage_endpoints/ai_usage_chat.py +++ b/litellm/proxy/management_endpoints/usage_endpoints/ai_usage_chat.py @@ -6,9 +6,12 @@ usage/spend data by querying the aggregated daily activity endpoints. import json from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Mapping, Sequence from datetime import date +from itertools import chain +from types import MappingProxyType from typing import TYPE_CHECKING, Final, Literal, NamedTuple, Protocol, TypeAlias, cast, overload from fastapi import HTTPException +from pydantic import BaseModel, ConfigDict from typing_extensions import ReadOnly, TypedDict import litellm @@ -470,7 +473,11 @@ def _resolve_model(model: str | None) -> str: _RouterRoute: TypeAlias = Literal["model_group", "deployment"] -def _router_route(router: "Router", model: str, caller_is_proxy_admin: bool) -> _RouterRoute | None: +def _router_route( + router: "Router", model: str, team_id: str | None, caller_is_proxy_admin: bool +) -> _RouterRoute | None: + if team_id is not None and router.map_team_model(model, team_id) is not None: + return "model_group" if router.is_recognized_model(model) or (caller_is_proxy_admin and model in router.team_public_model_names): return "model_group" if router.router_general_settings.pass_through_all_models: @@ -480,6 +487,49 @@ def _router_route(router: "Router", model: str, caller_is_proxy_admin: bool) -> return "deployment" if model in router.deployment_names else None +def _billed_caller(user_api_key_dict: "UserAPIKeyAuth") -> "UserAPIKeyAuth": + from litellm.constants import UI_SESSION_TOKEN_TEAM_ID + + if user_api_key_dict.team_id != UI_SESSION_TOKEN_TEAM_ID: + return user_api_key_dict + return user_api_key_dict.model_copy(update={"team_id": None}) + + +class _TaggedMetadata(BaseModel): + model_config = ConfigDict(frozen=True) + + tags: tuple[object, ...] | None = None + + +class _CallerTagSources(BaseModel): + model_config = ConfigDict(frozen=True, from_attributes=True) + + metadata: _TaggedMetadata | None = None + team_metadata: _TaggedMetadata | None = None + project_metadata: _TaggedMetadata | None = None + + +def _caller_routing_tags(user_api_key_dict: "UserAPIKeyAuth") -> tuple[str, ...]: + sources: Final = _CallerTagSources.model_validate(user_api_key_dict, from_attributes=True) + tags: Final = chain.from_iterable( + source.tags or () for source in (sources.metadata, sources.team_metadata, sources.project_metadata) if source + ) + return tuple(dict.fromkeys(tag for tag in tags if isinstance(tag, str))) + + +def _caller_metadata(caller: "UserAPIKeyAuth") -> Mapping[str, object]: + from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup + + billing_metadata: Final = cast( # cast-ok: the shared helper returns the request dict untyped + "Mapping[str, object]", + LiteLLMProxyRequestSetup.add_user_api_key_auth_to_request_metadata( # pyright: ignore[reportUnknownMemberType] # the shared billing helper takes and returns a bare dict + data={"metadata": {}}, user_api_key_dict=caller, _metadata_variable_name="metadata" + )["metadata"], + ) + caller_tags: Final = _caller_routing_tags(caller) + return MappingProxyType({**billing_metadata, **({"tags": list(caller_tags)} if caller_tags else {})}) + + async def _acompletion( model: str, messages: list[Mapping[str, object]], # mutable-ok: callers pass the request message list unchanged @@ -488,10 +538,7 @@ async def _acompletion( user_api_key_dict: "UserAPIKeyAuth | None" = None, ) -> ModelResponse | CustomStreamWrapper: from litellm.proxy.auth.auth_checks import key_model_aliases_for_auth_check - from litellm.proxy.litellm_pre_call_utils import ( - resolve_team_model_alias, - sanitized_user_api_key_auth, - ) + from litellm.proxy.litellm_pre_call_utils import resolve_team_model_alias from litellm.proxy.proxy_server import llm_router from litellm.router_utils.common_utils import is_proxy_admin_request from litellm.types.llms.openai import AllMessageValues @@ -508,11 +555,17 @@ async def _acompletion( resolved_model: Final = ( model_aliases.get(team_aliased_model, team_aliased_model) if model_aliases is not None else team_aliased_model ) - router_metadata: Final = ( - {"user_api_key_auth": sanitized_user_api_key_auth(user_api_key_dict)} if user_api_key_dict is not None else {} + caller: Final = _billed_caller(user_api_key_dict) if user_api_key_dict is not None else None + caller_metadata: Final[Mapping[str, object]] = ( + _caller_metadata(caller) if caller is not None else MappingProxyType({}) ) route: Final = ( - _router_route(llm_router, resolved_model, is_proxy_admin_request({"metadata": router_metadata})) + _router_route( + llm_router, + resolved_model, + caller.team_id if caller is not None else None, + is_proxy_admin_request({"metadata": caller_metadata}), + ) if llm_router is not None else None ) @@ -526,7 +579,7 @@ async def _acompletion( temperature=USAGE_AI_TEMPERATURE, drop_params=True, allowed_openai_params=list(USAGE_AI_REQUIRED_PARAMS), - metadata=router_metadata, + metadata=dict(caller_metadata), specific_deployment=route == "deployment", ) return await llm_router.acompletion( # pyright: ignore[reportUnknownMemberType] # Router.acompletion overloads carry untyped **kwargs @@ -536,7 +589,7 @@ async def _acompletion( temperature=USAGE_AI_TEMPERATURE, drop_params=True, allowed_openai_params=list(USAGE_AI_REQUIRED_PARAMS), - metadata=router_metadata, + metadata=dict(caller_metadata), specific_deployment=route == "deployment", ) return await litellm.acompletion( @@ -547,15 +600,24 @@ async def _acompletion( temperature=USAGE_AI_TEMPERATURE, drop_params=True, allowed_openai_params=list(USAGE_AI_REQUIRED_PARAMS), + metadata=dict(caller_metadata), ) async def check_usage_ai_chat_model_access(user_api_key_dict: "UserAPIKeyAuth", model: str | None) -> None: from litellm.proxy._types import ProxyException - from litellm.proxy.auth.auth_checks import can_key_call_resolved_model + from litellm.proxy.auth.auth_checks import ( + _personal_key_owner, # pyright: ignore[reportPrivateUsage] # the same owner lookup the resolved-model check uses + can_key_call_resolved_model, + can_user_call_model, + key_model_aliases_for_auth_check, + ) from litellm.proxy.proxy_server import ( llm_model_list, # pyright: ignore[reportUnknownVariableType] # proxy_server.llm_model_list is an untyped module global llm_router, + prisma_client, + proxy_logging_obj, + user_api_key_cache, ) resolved_model: Final = _resolve_model(model) @@ -566,14 +628,23 @@ async def check_usage_ai_chat_model_access(user_api_key_dict: "UserAPIKeyAuth", valid_token=user_api_key_dict, llm_router=llm_router, ) + if user_api_key_dict.team_id is None: + await can_user_call_model( + model=resolved_model, + llm_router=llm_router, + user_object=await _personal_key_owner( + valid_token=user_api_key_dict, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ), + key_model_aliases=key_model_aliases_for_auth_check(user_api_key_dict), + ) except ProxyException as denial: if (model or "").strip(): raise raise ProxyException( - message=( - f"Ask AI uses {resolved_model} when no model is picked, and this key cannot call it. " - f"Pick a model this key can call. {denial.message}" - ), + message=f"No model was picked, so Ask AI used its default. {denial.message} Pick a model you can call.", type=denial.type, param="model", code=denial.code, diff --git a/tests/unit/proxy/management_endpoints/usage_endpoints/test_ai_usage_chat.py b/tests/unit/proxy/management_endpoints/usage_endpoints/test_ai_usage_chat.py index 900ca8f9139..26ec9cc6659 100644 --- a/tests/unit/proxy/management_endpoints/usage_endpoints/test_ai_usage_chat.py +++ b/tests/unit/proxy/management_endpoints/usage_endpoints/test_ai_usage_chat.py @@ -12,6 +12,7 @@ import pytest import respx from fastapi import HTTPException +from litellm.proxy._types import LitellmUserRoles from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import ( TOOL_HANDLERS, TOOLS_ADMIN, @@ -1259,6 +1260,43 @@ class TestUsageAiChatModelAccess: assert exc_info.value.code == "403" + @pytest.mark.asyncio + @pytest.mark.parametrize(("model", "denied"), [(None, True), ("shared-model", False)]) + async def test_default_model_gets_the_user_model_check_a_picked_model_gets(self, model: str | None, denied: bool): + from litellm.proxy._types import LiteLLM_UserTable, ProxyException, UserAPIKeyAuth + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import ( + check_usage_ai_chat_model_access, + ) + + user_cache: Final = UserApiKeyCache() + await user_cache.async_set_cache( + key="u1", value=LiteLLM_UserTable(user_id="u1", models=["shared-model"]), model_type=LiteLLM_UserTable + ) + router: Final = self._router() + with ( + patch( # test-quality-ok: check reads the proxy_server.llm_router module global; no injection seam + "litellm.proxy.proxy_server.llm_router", router + ), + patch( # test-quality-ok: check reads the proxy_server.llm_model_list module global; no injection seam + "litellm.proxy.proxy_server.llm_model_list", None + ), + patch( # test-quality-ok: check reads the proxy_server.prisma_client module global; no injection seam + "litellm.proxy.proxy_server.prisma_client", None + ), + patch( # test-quality-ok: check reads the proxy_server.user_api_key_cache module global; no injection seam + "litellm.proxy.proxy_server.user_api_key_cache", user_cache + ), + ): + personal_key: Final = UserAPIKeyAuth(models=[], user_id="u1", api_key="sk-test") + if denied: + with pytest.raises(ProxyException) as exc_info: + await check_usage_ai_chat_model_access(user_api_key_dict=personal_key, model=model) + assert exc_info.value.code == "403" + assert "No model was picked" in exc_info.value.message + else: + assert await check_usage_ai_chat_model_access(user_api_key_dict=personal_key, model=model) is None + class TestUsageAiChatTeamModelRouting: @staticmethod @@ -1301,22 +1339,105 @@ class TestUsageAiChatTeamModelRouting: assert result.choices[0].message.content == "from the team deployment" @pytest.mark.asyncio - async def test_team_key_ask_ai_spend_is_not_attributed_to_the_team(self, monkeypatch: pytest.MonkeyPatch): + @pytest.mark.parametrize("user_role", [LitellmUserRoles.INTERNAL_USER, LitellmUserRoles.PROXY_ADMIN]) + async def test_team_key_reaches_its_own_team_deployment_when_teams_share_a_public_name( + self, user_role: LitellmUserRoles + ): + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import ( + _acompletion, + ) + from litellm.router import Router + + router: Final = Router( + model_list=[ + { + "model_name": f"model_name_{team_id}_internal", + "litellm_params": { + "model": "openai/gpt-4o", + "api_key": "sk-dummy", + "mock_response": f"from the {team_id} deployment", + }, + "model_info": {"team_id": team_id, "team_public_model_name": "shared-team-model"}, + } + for team_id in ("team-a", "team-b") + ] + ) + + with patch( # test-quality-ok: _acompletion reads the proxy_server.llm_router module global; no injection seam + "litellm.proxy.proxy_server.llm_router", router + ): + result: Final = await _acompletion( + model="shared-team-model", + messages=[{"role": "user", "content": "hi"}], + user_api_key_dict=UserAPIKeyAuth(user_role=user_role, team_id="team-b", api_key="sk-test"), + ) + + assert result.choices[0].message.content == "from the team-b deployment" + + @pytest.mark.asyncio + async def test_key_and_team_tags_pick_the_tagged_deployment(self): + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import ( + _acompletion, + ) + from litellm.router import Router + + router: Final = Router( + model_list=[ + { + "model_name": "tagged-model", + "litellm_params": { + "model": "openai/gpt-4o", + "api_key": "sk-dummy", + "mock_response": f"from the {tag} deployment", + "tags": [tag], + }, + } + for tag in ("team-tag", "default") + ], + enable_tag_filtering=True, + ) + + with patch( # test-quality-ok: _acompletion reads the proxy_server.llm_router module global; no injection seam + "litellm.proxy.proxy_server.llm_router", router + ): + result: Final = await _acompletion( + model="tagged-model", + messages=[{"role": "user", "content": "hi"}], + user_api_key_dict=UserAPIKeyAuth( + team_id="team-a", team_metadata={"tags": ["team-tag"]}, metadata={}, api_key="sk-test" + ), + ) + + assert result.choices[0].message.content == "from the team-tag deployment" + + @pytest.mark.asyncio + @pytest.mark.parametrize(("team_id", "billed_team_id"), [("team-a", "team-a"), ("litellm-dashboard", None)]) + async def test_ask_ai_spend_is_billed_to_the_calling_key_user_and_team( + self, monkeypatch: pytest.MonkeyPatch, team_id: str, billed_team_id: str | None + ): import asyncio import litellm from litellm.integrations.custom_logger import CustomLogger - from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import ( _acompletion, ) - logged: Final[asyncio.Queue[object]] = asyncio.Queue() + logged: Final[asyncio.Queue[tuple[object, object, object]]] = asyncio.Queue() class SpendCapture(CustomLogger): async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): - await logged.put(kwargs["standard_logging_object"]["metadata"]["user_api_key_team_id"]) + metadata = kwargs["standard_logging_object"]["metadata"] + await logged.put( + (metadata["user_api_key_hash"], metadata["user_api_key_user_id"], metadata["user_api_key_team_id"]) + ) + caller: Final = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, team_id=team_id, user_id="u1", api_key="sk-test" + ) monkeypatch.setattr(litellm, "callbacks", [SpendCapture()]) with patch( # test-quality-ok: _acompletion reads the proxy_server.llm_router module global; no injection seam "litellm.proxy.proxy_server.llm_router", self._team_router() @@ -1324,12 +1445,10 @@ class TestUsageAiChatTeamModelRouting: await _acompletion( model="my-team-model", messages=[{"role": "user", "content": "hi"}], - user_api_key_dict=UserAPIKeyAuth( - user_role=LitellmUserRoles.PROXY_ADMIN, team_id="team-a", api_key="sk-test" - ), + user_api_key_dict=caller, ) - assert await asyncio.wait_for(logged.get(), timeout=10) is None + assert await asyncio.wait_for(logged.get(), timeout=10) == (caller.api_key, "u1", billed_team_id) @pytest.mark.asyncio async def test_team_public_model_name_without_team_id_uses_litellm(self):