From 982d0bb4ae6c6c1726410b548edbae665afd4f1c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 05:15:15 -0700 Subject: [PATCH] fix(proxy): bill no Admin UI session for Ask AI, apply every alias layer, and ignore non-list tags --- .../usage_endpoints/ai_usage_chat.py | 92 +++++++++------ .../usage_endpoints/endpoints.py | 4 +- .../usage_endpoints/test_ai_usage_chat.py | 110 +++++++++++++----- 3 files changed, 139 insertions(+), 67 deletions(-) diff --git a/litellm/proxy/management_endpoints/usage_endpoints/ai_usage_chat.py b/litellm/proxy/management_endpoints/usage_endpoints/ai_usage_chat.py index fa1f22660b1..7355d71f8bd 100644 --- a/litellm/proxy/management_endpoints/usage_endpoints/ai_usage_chat.py +++ b/litellm/proxy/management_endpoints/usage_endpoints/ai_usage_chat.py @@ -11,12 +11,12 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Final, Literal, NamedTuple, Protocol, TypeAlias, cast, overload from fastapi import HTTPException -from pydantic import BaseModel, ConfigDict +from pydantic import BaseModel, ConfigDict, field_validator from typing_extensions import ReadOnly, TypedDict import litellm from litellm._logging import verbose_proxy_logger -from litellm.constants import DEFAULT_COMPETITOR_DISCOVERY_MODEL +from litellm.constants import DEFAULT_COMPETITOR_DISCOVERY_MODEL, UI_SESSION_TOKEN_TEAM_ID from litellm.proxy._types import CommonProxyErrors from litellm.types.proxy.management_endpoints.common_daily_activity import ( SpendAnalyticsPaginatedResponse, @@ -487,19 +487,16 @@ def _router_route( return "deployment" if model in router.deployment_names else None -def _billed_caller(user_api_key_dict: "UserAPIKeyAuth") -> "UserAPIKeyAuth": - from litellm.constants import UI_SESSION_TOKEN_TEAM_ID - - if user_api_key_dict.team_id != UI_SESSION_TOKEN_TEAM_ID: - return user_api_key_dict - return user_api_key_dict.model_copy(update={"team_id": None}) - - class _TaggedMetadata(BaseModel): model_config = ConfigDict(frozen=True) tags: tuple[object, ...] | None = None + @field_validator("tags", mode="before") + @classmethod + def _ignore_non_list_tags(cls, tags: object) -> object: + return tags if isinstance(tags, (list, tuple)) else None + class _CallerTagSources(BaseModel): model_config = ConfigDict(frozen=True, from_attributes=True) @@ -517,17 +514,54 @@ def _caller_routing_tags(user_api_key_dict: "UserAPIKeyAuth") -> tuple[str, ...] return tuple(dict.fromkeys(tag for tag in tags if isinstance(tag, str))) -def _caller_metadata(caller: "UserAPIKeyAuth") -> Mapping[str, object]: - from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup +def _caller_identity_metadata(user_api_key_dict: "UserAPIKeyAuth") -> Mapping[str, object]: + from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup, sanitized_user_api_key_auth - billing_metadata: Final = cast( # cast-ok: the shared helper returns the request dict untyped + if user_api_key_dict.team_id == UI_SESSION_TOKEN_TEAM_ID: + return MappingProxyType({"user_api_key_auth": sanitized_user_api_key_auth(user_api_key_dict)}) + return cast( # cast-ok: the shared helper returns the request dict untyped "Mapping[str, object]", LiteLLMProxyRequestSetup.add_user_api_key_auth_to_request_metadata( # pyright: ignore[reportUnknownMemberType] # the shared billing helper takes and returns a bare dict - data={"metadata": {}}, user_api_key_dict=caller, _metadata_variable_name="metadata" + data={"metadata": {}}, user_api_key_dict=user_api_key_dict, _metadata_variable_name="metadata" )["metadata"], ) - caller_tags: Final = _caller_routing_tags(caller) - return MappingProxyType({**billing_metadata, **({"tags": list(caller_tags)} if caller_tags else {})}) + + +def _caller_metadata(user_api_key_dict: "UserAPIKeyAuth") -> Mapping[str, object]: + caller_tags: Final = _caller_routing_tags(user_api_key_dict) + return MappingProxyType( + {**_caller_identity_metadata(user_api_key_dict), **({"tags": list(caller_tags)} if caller_tags else {})} + ) + + +async def resolve_usage_ai_chat_model(user_api_key_dict: "UserAPIKeyAuth", model: str | None) -> str: + from litellm.proxy.auth.auth_checks import key_model_aliases_for_auth_check + from litellm.proxy.common_request_processing import ( + _resolve_per_request_model_group_alias, # pyright: ignore[reportPrivateUsage] # the key and team alias rule /chat/completions applies + ) + from litellm.proxy.litellm_pre_call_utils import resolve_team_model_alias + from litellm.proxy.proxy_server import llm_router, prisma_client, proxy_config, proxy_logging_obj + + team_aliased_model: Final = resolve_team_model_alias(_resolve_model(model), user_api_key_dict) + global_aliased_model: Final = litellm.model_alias_map.get(team_aliased_model, team_aliased_model) + key_aliases: Final = key_model_aliases_for_auth_check(user_api_key_dict) + key_aliased_model: Final = ( + key_aliases.get(global_aliased_model, global_aliased_model) if key_aliases is not None else global_aliased_model + ) + if llm_router is None: + return key_aliased_model + router_settings: Final = await proxy_config.get_hierarchical_router_settings( + user_api_key_dict=user_api_key_dict, prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj + ) + if router_settings is None: + return key_aliased_model + group_alias_target: Final = await _resolve_per_request_model_group_alias( + requested_model=key_aliased_model, + router_settings=router_settings, + user_api_key_dict=user_api_key_dict, + llm_router=llm_router, + ) + return group_alias_target or key_aliased_model async def _acompletion( @@ -537,33 +571,21 @@ async def _acompletion( stream: bool = False, user_api_key_dict: "UserAPIKeyAuth | None" = None, ) -> ModelResponse | CustomStreamWrapper: - from litellm.proxy.auth.auth_checks import key_model_aliases_for_auth_check - from litellm.proxy.litellm_pre_call_utils import resolve_team_model_alias from litellm.proxy.proxy_server import llm_router - from litellm.router_utils.common_utils import is_proxy_admin_request + from litellm.router_utils.common_utils import get_request_team_id, is_proxy_admin_request from litellm.types.llms.openai import AllMessageValues completion_messages: Final = cast( # cast-ok: acompletion accepts the same Mapping messages it already receives list[AllMessageValues], messages ) - team_aliased_model: Final = ( - resolve_team_model_alias(model, user_api_key_dict) if user_api_key_dict is not None else model - ) - model_aliases: Final = ( - key_model_aliases_for_auth_check(user_api_key_dict) if user_api_key_dict is not None else None - ) - resolved_model: Final = ( - model_aliases.get(team_aliased_model, team_aliased_model) if model_aliases is not None else team_aliased_model - ) - caller: Final = _billed_caller(user_api_key_dict) if user_api_key_dict is not None else None caller_metadata: Final[Mapping[str, object]] = ( - _caller_metadata(caller) if caller is not None else MappingProxyType({}) + _caller_metadata(user_api_key_dict) if user_api_key_dict is not None else MappingProxyType({}) ) route: Final = ( _router_route( llm_router, - resolved_model, - caller.team_id if caller is not None else None, + model, + get_request_team_id({"metadata": caller_metadata}), is_proxy_admin_request({"metadata": caller_metadata}), ) if llm_router is not None @@ -572,7 +594,7 @@ async def _acompletion( if llm_router is not None and route is not None: if stream: return await llm_router.acompletion( # pyright: ignore[reportUnknownMemberType] # Router.acompletion overloads carry untyped **kwargs - model=resolved_model, + model=model, messages=completion_messages, tools=tools, stream=True, @@ -583,7 +605,7 @@ async def _acompletion( specific_deployment=route == "deployment", ) return await llm_router.acompletion( # pyright: ignore[reportUnknownMemberType] # Router.acompletion overloads carry untyped **kwargs - model=resolved_model, + model=model, messages=completion_messages, tools=tools, temperature=USAGE_AI_TEMPERATURE, @@ -593,7 +615,7 @@ async def _acompletion( specific_deployment=route == "deployment", ) return await litellm.acompletion( - model=resolved_model, + model=model, messages=completion_messages, tools=tools, stream=stream, diff --git a/litellm/proxy/management_endpoints/usage_endpoints/endpoints.py b/litellm/proxy/management_endpoints/usage_endpoints/endpoints.py index 021ad3fb938..88f2cd77dad 100644 --- a/litellm/proxy/management_endpoints/usage_endpoints/endpoints.py +++ b/litellm/proxy/management_endpoints/usage_endpoints/endpoints.py @@ -52,6 +52,7 @@ async def usage_ai_chat( ) from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import ( check_usage_ai_chat_model_access, + resolve_usage_ai_chat_model, stream_usage_ai_chat, ) @@ -63,12 +64,13 @@ async def usage_ai_chat( messages: Final = [{"role": m.role, "content": m.content} for m in data.messages] await check_usage_ai_chat_model_access(user_api_key_dict=user_api_key_dict, model=data.model) + routed_model: Final = await resolve_usage_ai_chat_model(user_api_key_dict=user_api_key_dict, model=data.model) return StreamingResponse( wrap_sse_stream_with_keepalive_pings( stream_usage_ai_chat( messages=messages, - model=data.model, + model=routed_model, user_id=user_id, is_admin=is_admin, user_api_key_dict=user_api_key_dict, diff --git a/tests/unit/proxy/management_endpoints/usage_endpoints/test_ai_usage_chat.py b/tests/unit/proxy/management_endpoints/usage_endpoints/test_ai_usage_chat.py index 26ec9cc6659..c4cd3ce1dfc 100644 --- a/tests/unit/proxy/management_endpoints/usage_endpoints/test_ai_usage_chat.py +++ b/tests/unit/proxy/management_endpoints/usage_endpoints/test_ai_usage_chat.py @@ -12,7 +12,7 @@ import pytest import respx from fastapi import HTTPException -from litellm.proxy._types import LitellmUserRoles +from litellm.proxy._types import LitellmUserRoles, hash_token from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import ( TOOL_HANDLERS, TOOLS_ADMIN, @@ -24,6 +24,7 @@ from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import ( ) if TYPE_CHECKING: + from litellm.proxy._types import UserAPIKeyAuth from litellm.router import Router @@ -1376,7 +1377,8 @@ class TestUsageAiChatTeamModelRouting: assert result.choices[0].message.content == "from the team-b deployment" @pytest.mark.asyncio - async def test_key_and_team_tags_pick_the_tagged_deployment(self): + @pytest.mark.parametrize("key_metadata", [{}, {"tags": "prod"}, {"tags": {"env": "prod"}}]) + async def test_key_and_team_tags_pick_the_tagged_deployment(self, key_metadata: dict[str, object]): from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import ( _acompletion, @@ -1406,16 +1408,19 @@ class TestUsageAiChatTeamModelRouting: model="tagged-model", messages=[{"role": "user", "content": "hi"}], user_api_key_dict=UserAPIKeyAuth( - team_id="team-a", team_metadata={"tags": ["team-tag"]}, metadata={}, api_key="sk-test" + team_id="team-a", team_metadata={"tags": ["team-tag"]}, metadata=key_metadata, api_key="sk-test" ), ) assert result.choices[0].message.content == "from the team-tag deployment" @pytest.mark.asyncio - @pytest.mark.parametrize(("team_id", "billed_team_id"), [("team-a", "team-a"), ("litellm-dashboard", None)]) - async def test_ask_ai_spend_is_billed_to_the_calling_key_user_and_team( - self, monkeypatch: pytest.MonkeyPatch, team_id: str, billed_team_id: str | None + @pytest.mark.parametrize( + ("team_id", "billed_caller"), + [("team-a", (hash_token("sk-test"), "u1", "team-a")), ("litellm-dashboard", (None, None, None))], + ) + async def test_ask_ai_bills_an_api_key_caller_and_never_an_admin_ui_session( + self, monkeypatch: pytest.MonkeyPatch, team_id: str, billed_caller: tuple[str | None, str | None, str | None] ): import asyncio @@ -1448,7 +1453,7 @@ class TestUsageAiChatTeamModelRouting: user_api_key_dict=caller, ) - assert await asyncio.wait_for(logged.get(), timeout=10) == (caller.api_key, "u1", billed_team_id) + assert await asyncio.wait_for(logged.get(), timeout=10) == billed_caller @pytest.mark.asyncio async def test_team_public_model_name_without_team_id_uses_litellm(self): @@ -1493,45 +1498,86 @@ class TestUsageAiChatTeamModelRouting: mock_litellm_acompletion.assert_awaited_once() router.acompletion.assert_not_called() - @pytest.mark.asyncio - async def test_key_alias_resolves_to_router_model(self): - from litellm.proxy._types import UserAPIKeyAuth - from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import ( - _acompletion, + @staticmethod + async def _endpoint_body(user_api_key_dict: "UserAPIKeyAuth") -> bytes: + from litellm.proxy.management_endpoints.usage_endpoints.endpoints import ( + ChatMessage, + UsageAIChatRequest, + usage_ai_chat, ) from litellm.router import Router router: Final = Router( model_list=[ { - "model_name": "gpt-4o-mini", - "litellm_params": {"model": "gpt-4o-mini", "api_key": "sk-dummy"}, + "model_name": "prod-model", + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_key": "sk-dummy", + "mock_response": "from the aliased deployment", + }, } ] ) - expected_response: Final = MagicMock() - router.acompletion = AsyncMock(return_value=expected_response) - mock_litellm_acompletion = AsyncMock(return_value=MagicMock()) - with ( - patch( # test-quality-ok: _acompletion reads the proxy_server.llm_router module global; no injection seam + patch( # test-quality-ok: Ask AI reads the proxy_server.llm_router module global; no injection seam "litellm.proxy.proxy_server.llm_router", router ), - patch( # test-quality-ok: _acompletion calls the module-level litellm.acompletion directly; no injection seam - "litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.litellm.acompletion", - new=mock_litellm_acompletion, + patch( # test-quality-ok: the check reads the proxy_server.llm_model_list module global; no injection seam + "litellm.proxy.proxy_server.llm_model_list", router.model_list + ), + patch( # test-quality-ok: the check reads the proxy_server.prisma_client module global; no injection seam + "litellm.proxy.proxy_server.prisma_client", None ), ): - result: Final = await _acompletion( - model="my-alias", - messages=[{"role": "user", "content": "hi"}], - user_api_key_dict=UserAPIKeyAuth(aliases={"my-alias": "gpt-4o-mini"}), + response: Final = await usage_ai_chat( + data=UsageAIChatRequest(messages=[ChatMessage(role="user", content="hi")], model="prod-alias"), + request=MagicMock(), + user_api_key_dict=user_api_key_dict, + ) + return b"".join( + [chunk if isinstance(chunk, bytes) else chunk.encode() async for chunk in response.body_iterator] ) - assert result is expected_response - router.acompletion.assert_awaited_once() - assert router.acompletion.await_args.kwargs["model"] == "gpt-4o-mini" - mock_litellm_acompletion.assert_not_called() + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("key_settings", "global_aliases"), + [ + ({"aliases": {"prod-alias": "prod-model"}}, {}), + ({}, {"prod-alias": "prod-model"}), + ({"router_settings": {"model_group_alias": {"prod-alias": "prod-model"}}}, {}), + ], + ) + async def test_every_alias_layer_routes_to_its_target( + self, key_settings: dict[str, object], global_aliases: dict[str, str] + ): + import litellm + from litellm.proxy._types import UserAPIKeyAuth + + admin_key: Final = UserAPIKeyAuth( + user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-test", **key_settings + ) + with patch.dict(litellm.model_alias_map, global_aliases): + body: Final = await self._endpoint_body(admin_key) + + assert b'"content": "from the aliased deployment"' in body + assert b'"type": "error"' not in body + + @pytest.mark.asyncio + async def test_group_alias_to_a_model_the_key_cannot_call_is_denied_before_streaming(self): + from litellm.proxy._types import ProxyException, UserAPIKeyAuth + + limited_key: Final = UserAPIKeyAuth( + user_id="admin", + user_role=LitellmUserRoles.PROXY_ADMIN, + api_key="sk-test", + models=["prod-alias"], + router_settings={"model_group_alias": {"prod-alias": "prod-model"}}, + ) + with pytest.raises(ProxyException) as exc_info: + await self._endpoint_body(limited_key) + + assert exc_info.value.code == "403" @pytest.mark.asyncio async def test_router_calls_receive_sanitized_caller_auth_for_access_group_filtering(self): @@ -1586,6 +1632,7 @@ class TestUsageAiChatTeamModelRouting: from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import ( DEFAULT_COMPETITOR_DISCOVERY_MODEL, check_usage_ai_chat_model_access, + resolve_usage_ai_chat_model, ) from litellm.router import Router @@ -1628,11 +1675,12 @@ class TestUsageAiChatTeamModelRouting: ), ): await check_usage_ai_chat_model_access(user_api_key_dict=team_key, model=None) + routed_model: Final = await resolve_usage_ai_chat_model(user_api_key_dict=team_key, model=None) events: Final = [ json.loads(raw.removeprefix("data: ")) async for raw in stream_usage_ai_chat( messages=[{"role": "user", "content": "hi"}], - model=None, + model=routed_model, is_admin=True, user_api_key_dict=team_key, )