fix(proxy): bill no Admin UI session for Ask AI, apply every alias layer, and ignore non-list tags

This commit is contained in:
mateo-berri 2026-10-20 05:15:15 -07:00
parent 67f701fdf4
commit 982d0bb4ae
3 changed files with 139 additions and 67 deletions

View file

@ -11,12 +11,12 @@ from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Literal, NamedTuple, Protocol, TypeAlias, cast, overload
from fastapi import HTTPException
from pydantic import BaseModel, ConfigDict
from pydantic import BaseModel, ConfigDict, field_validator
from typing_extensions import ReadOnly, TypedDict
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.constants import DEFAULT_COMPETITOR_DISCOVERY_MODEL
from litellm.constants import DEFAULT_COMPETITOR_DISCOVERY_MODEL, UI_SESSION_TOKEN_TEAM_ID
from litellm.proxy._types import CommonProxyErrors
from litellm.types.proxy.management_endpoints.common_daily_activity import (
SpendAnalyticsPaginatedResponse,
@ -487,19 +487,16 @@ def _router_route(
return "deployment" if model in router.deployment_names else None
def _billed_caller(user_api_key_dict: "UserAPIKeyAuth") -> "UserAPIKeyAuth":
from litellm.constants import UI_SESSION_TOKEN_TEAM_ID
if user_api_key_dict.team_id != UI_SESSION_TOKEN_TEAM_ID:
return user_api_key_dict
return user_api_key_dict.model_copy(update={"team_id": None})
class _TaggedMetadata(BaseModel):
model_config = ConfigDict(frozen=True)
tags: tuple[object, ...] | None = None
@field_validator("tags", mode="before")
@classmethod
def _ignore_non_list_tags(cls, tags: object) -> object:
return tags if isinstance(tags, (list, tuple)) else None
class _CallerTagSources(BaseModel):
model_config = ConfigDict(frozen=True, from_attributes=True)
@ -517,17 +514,54 @@ def _caller_routing_tags(user_api_key_dict: "UserAPIKeyAuth") -> tuple[str, ...]
return tuple(dict.fromkeys(tag for tag in tags if isinstance(tag, str)))
def _caller_metadata(caller: "UserAPIKeyAuth") -> Mapping[str, object]:
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup
def _caller_identity_metadata(user_api_key_dict: "UserAPIKeyAuth") -> Mapping[str, object]:
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup, sanitized_user_api_key_auth
billing_metadata: Final = cast( # cast-ok: the shared helper returns the request dict untyped
if user_api_key_dict.team_id == UI_SESSION_TOKEN_TEAM_ID:
return MappingProxyType({"user_api_key_auth": sanitized_user_api_key_auth(user_api_key_dict)})
return cast( # cast-ok: the shared helper returns the request dict untyped
"Mapping[str, object]",
LiteLLMProxyRequestSetup.add_user_api_key_auth_to_request_metadata( # pyright: ignore[reportUnknownMemberType] # the shared billing helper takes and returns a bare dict
data={"metadata": {}}, user_api_key_dict=caller, _metadata_variable_name="metadata"
data={"metadata": {}}, user_api_key_dict=user_api_key_dict, _metadata_variable_name="metadata"
)["metadata"],
)
caller_tags: Final = _caller_routing_tags(caller)
return MappingProxyType({**billing_metadata, **({"tags": list(caller_tags)} if caller_tags else {})})
def _caller_metadata(user_api_key_dict: "UserAPIKeyAuth") -> Mapping[str, object]:
caller_tags: Final = _caller_routing_tags(user_api_key_dict)
return MappingProxyType(
{**_caller_identity_metadata(user_api_key_dict), **({"tags": list(caller_tags)} if caller_tags else {})}
)
async def resolve_usage_ai_chat_model(user_api_key_dict: "UserAPIKeyAuth", model: str | None) -> str:
from litellm.proxy.auth.auth_checks import key_model_aliases_for_auth_check
from litellm.proxy.common_request_processing import (
_resolve_per_request_model_group_alias, # pyright: ignore[reportPrivateUsage] # the key and team alias rule /chat/completions applies
)
from litellm.proxy.litellm_pre_call_utils import resolve_team_model_alias
from litellm.proxy.proxy_server import llm_router, prisma_client, proxy_config, proxy_logging_obj
team_aliased_model: Final = resolve_team_model_alias(_resolve_model(model), user_api_key_dict)
global_aliased_model: Final = litellm.model_alias_map.get(team_aliased_model, team_aliased_model)
key_aliases: Final = key_model_aliases_for_auth_check(user_api_key_dict)
key_aliased_model: Final = (
key_aliases.get(global_aliased_model, global_aliased_model) if key_aliases is not None else global_aliased_model
)
if llm_router is None:
return key_aliased_model
router_settings: Final = await proxy_config.get_hierarchical_router_settings(
user_api_key_dict=user_api_key_dict, prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj
)
if router_settings is None:
return key_aliased_model
group_alias_target: Final = await _resolve_per_request_model_group_alias(
requested_model=key_aliased_model,
router_settings=router_settings,
user_api_key_dict=user_api_key_dict,
llm_router=llm_router,
)
return group_alias_target or key_aliased_model
async def _acompletion(
@ -537,33 +571,21 @@ async def _acompletion(
stream: bool = False,
user_api_key_dict: "UserAPIKeyAuth | None" = None,
) -> ModelResponse | CustomStreamWrapper:
from litellm.proxy.auth.auth_checks import key_model_aliases_for_auth_check
from litellm.proxy.litellm_pre_call_utils import resolve_team_model_alias
from litellm.proxy.proxy_server import llm_router
from litellm.router_utils.common_utils import is_proxy_admin_request
from litellm.router_utils.common_utils import get_request_team_id, is_proxy_admin_request
from litellm.types.llms.openai import AllMessageValues
completion_messages: Final = cast( # cast-ok: acompletion accepts the same Mapping messages it already receives
list[AllMessageValues], messages
)
team_aliased_model: Final = (
resolve_team_model_alias(model, user_api_key_dict) if user_api_key_dict is not None else model
)
model_aliases: Final = (
key_model_aliases_for_auth_check(user_api_key_dict) if user_api_key_dict is not None else None
)
resolved_model: Final = (
model_aliases.get(team_aliased_model, team_aliased_model) if model_aliases is not None else team_aliased_model
)
caller: Final = _billed_caller(user_api_key_dict) if user_api_key_dict is not None else None
caller_metadata: Final[Mapping[str, object]] = (
_caller_metadata(caller) if caller is not None else MappingProxyType({})
_caller_metadata(user_api_key_dict) if user_api_key_dict is not None else MappingProxyType({})
)
route: Final = (
_router_route(
llm_router,
resolved_model,
caller.team_id if caller is not None else None,
model,
get_request_team_id({"metadata": caller_metadata}),
is_proxy_admin_request({"metadata": caller_metadata}),
)
if llm_router is not None
@ -572,7 +594,7 @@ async def _acompletion(
if llm_router is not None and route is not None:
if stream:
return await llm_router.acompletion( # pyright: ignore[reportUnknownMemberType] # Router.acompletion overloads carry untyped **kwargs
model=resolved_model,
model=model,
messages=completion_messages,
tools=tools,
stream=True,
@ -583,7 +605,7 @@ async def _acompletion(
specific_deployment=route == "deployment",
)
return await llm_router.acompletion( # pyright: ignore[reportUnknownMemberType] # Router.acompletion overloads carry untyped **kwargs
model=resolved_model,
model=model,
messages=completion_messages,
tools=tools,
temperature=USAGE_AI_TEMPERATURE,
@ -593,7 +615,7 @@ async def _acompletion(
specific_deployment=route == "deployment",
)
return await litellm.acompletion(
model=resolved_model,
model=model,
messages=completion_messages,
tools=tools,
stream=stream,

View file

@ -52,6 +52,7 @@ async def usage_ai_chat(
)
from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import (
check_usage_ai_chat_model_access,
resolve_usage_ai_chat_model,
stream_usage_ai_chat,
)
@ -63,12 +64,13 @@ async def usage_ai_chat(
messages: Final = [{"role": m.role, "content": m.content} for m in data.messages]
await check_usage_ai_chat_model_access(user_api_key_dict=user_api_key_dict, model=data.model)
routed_model: Final = await resolve_usage_ai_chat_model(user_api_key_dict=user_api_key_dict, model=data.model)
return StreamingResponse(
wrap_sse_stream_with_keepalive_pings(
stream_usage_ai_chat(
messages=messages,
model=data.model,
model=routed_model,
user_id=user_id,
is_admin=is_admin,
user_api_key_dict=user_api_key_dict,

View file

@ -12,7 +12,7 @@ import pytest
import respx
from fastapi import HTTPException
from litellm.proxy._types import LitellmUserRoles
from litellm.proxy._types import LitellmUserRoles, hash_token
from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import (
TOOL_HANDLERS,
TOOLS_ADMIN,
@ -24,6 +24,7 @@ from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import (
)
if TYPE_CHECKING:
from litellm.proxy._types import UserAPIKeyAuth
from litellm.router import Router
@ -1376,7 +1377,8 @@ class TestUsageAiChatTeamModelRouting:
assert result.choices[0].message.content == "from the team-b deployment"
@pytest.mark.asyncio
async def test_key_and_team_tags_pick_the_tagged_deployment(self):
@pytest.mark.parametrize("key_metadata", [{}, {"tags": "prod"}, {"tags": {"env": "prod"}}])
async def test_key_and_team_tags_pick_the_tagged_deployment(self, key_metadata: dict[str, object]):
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import (
_acompletion,
@ -1406,16 +1408,19 @@ class TestUsageAiChatTeamModelRouting:
model="tagged-model",
messages=[{"role": "user", "content": "hi"}],
user_api_key_dict=UserAPIKeyAuth(
team_id="team-a", team_metadata={"tags": ["team-tag"]}, metadata={}, api_key="sk-test"
team_id="team-a", team_metadata={"tags": ["team-tag"]}, metadata=key_metadata, api_key="sk-test"
),
)
assert result.choices[0].message.content == "from the team-tag deployment"
@pytest.mark.asyncio
@pytest.mark.parametrize(("team_id", "billed_team_id"), [("team-a", "team-a"), ("litellm-dashboard", None)])
async def test_ask_ai_spend_is_billed_to_the_calling_key_user_and_team(
self, monkeypatch: pytest.MonkeyPatch, team_id: str, billed_team_id: str | None
@pytest.mark.parametrize(
("team_id", "billed_caller"),
[("team-a", (hash_token("sk-test"), "u1", "team-a")), ("litellm-dashboard", (None, None, None))],
)
async def test_ask_ai_bills_an_api_key_caller_and_never_an_admin_ui_session(
self, monkeypatch: pytest.MonkeyPatch, team_id: str, billed_caller: tuple[str | None, str | None, str | None]
):
import asyncio
@ -1448,7 +1453,7 @@ class TestUsageAiChatTeamModelRouting:
user_api_key_dict=caller,
)
assert await asyncio.wait_for(logged.get(), timeout=10) == (caller.api_key, "u1", billed_team_id)
assert await asyncio.wait_for(logged.get(), timeout=10) == billed_caller
@pytest.mark.asyncio
async def test_team_public_model_name_without_team_id_uses_litellm(self):
@ -1493,45 +1498,86 @@ class TestUsageAiChatTeamModelRouting:
mock_litellm_acompletion.assert_awaited_once()
router.acompletion.assert_not_called()
@pytest.mark.asyncio
async def test_key_alias_resolves_to_router_model(self):
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import (
_acompletion,
@staticmethod
async def _endpoint_body(user_api_key_dict: "UserAPIKeyAuth") -> bytes:
from litellm.proxy.management_endpoints.usage_endpoints.endpoints import (
ChatMessage,
UsageAIChatRequest,
usage_ai_chat,
)
from litellm.router import Router
router: Final = Router(
model_list=[
{
"model_name": "gpt-4o-mini",
"litellm_params": {"model": "gpt-4o-mini", "api_key": "sk-dummy"},
"model_name": "prod-model",
"litellm_params": {
"model": "openai/gpt-4o-mini",
"api_key": "sk-dummy",
"mock_response": "from the aliased deployment",
},
}
]
)
expected_response: Final = MagicMock()
router.acompletion = AsyncMock(return_value=expected_response)
mock_litellm_acompletion = AsyncMock(return_value=MagicMock())
with (
patch( # test-quality-ok: _acompletion reads the proxy_server.llm_router module global; no injection seam
patch( # test-quality-ok: Ask AI reads the proxy_server.llm_router module global; no injection seam
"litellm.proxy.proxy_server.llm_router", router
),
patch( # test-quality-ok: _acompletion calls the module-level litellm.acompletion directly; no injection seam
"litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.litellm.acompletion",
new=mock_litellm_acompletion,
patch( # test-quality-ok: the check reads the proxy_server.llm_model_list module global; no injection seam
"litellm.proxy.proxy_server.llm_model_list", router.model_list
),
patch( # test-quality-ok: the check reads the proxy_server.prisma_client module global; no injection seam
"litellm.proxy.proxy_server.prisma_client", None
),
):
result: Final = await _acompletion(
model="my-alias",
messages=[{"role": "user", "content": "hi"}],
user_api_key_dict=UserAPIKeyAuth(aliases={"my-alias": "gpt-4o-mini"}),
response: Final = await usage_ai_chat(
data=UsageAIChatRequest(messages=[ChatMessage(role="user", content="hi")], model="prod-alias"),
request=MagicMock(),
user_api_key_dict=user_api_key_dict,
)
return b"".join(
[chunk if isinstance(chunk, bytes) else chunk.encode() async for chunk in response.body_iterator]
)
assert result is expected_response
router.acompletion.assert_awaited_once()
assert router.acompletion.await_args.kwargs["model"] == "gpt-4o-mini"
mock_litellm_acompletion.assert_not_called()
@pytest.mark.asyncio
@pytest.mark.parametrize(
("key_settings", "global_aliases"),
[
({"aliases": {"prod-alias": "prod-model"}}, {}),
({}, {"prod-alias": "prod-model"}),
({"router_settings": {"model_group_alias": {"prod-alias": "prod-model"}}}, {}),
],
)
async def test_every_alias_layer_routes_to_its_target(
self, key_settings: dict[str, object], global_aliases: dict[str, str]
):
import litellm
from litellm.proxy._types import UserAPIKeyAuth
admin_key: Final = UserAPIKeyAuth(
user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-test", **key_settings
)
with patch.dict(litellm.model_alias_map, global_aliases):
body: Final = await self._endpoint_body(admin_key)
assert b'"content": "from the aliased deployment"' in body
assert b'"type": "error"' not in body
@pytest.mark.asyncio
async def test_group_alias_to_a_model_the_key_cannot_call_is_denied_before_streaming(self):
from litellm.proxy._types import ProxyException, UserAPIKeyAuth
limited_key: Final = UserAPIKeyAuth(
user_id="admin",
user_role=LitellmUserRoles.PROXY_ADMIN,
api_key="sk-test",
models=["prod-alias"],
router_settings={"model_group_alias": {"prod-alias": "prod-model"}},
)
with pytest.raises(ProxyException) as exc_info:
await self._endpoint_body(limited_key)
assert exc_info.value.code == "403"
@pytest.mark.asyncio
async def test_router_calls_receive_sanitized_caller_auth_for_access_group_filtering(self):
@ -1586,6 +1632,7 @@ class TestUsageAiChatTeamModelRouting:
from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import (
DEFAULT_COMPETITOR_DISCOVERY_MODEL,
check_usage_ai_chat_model_access,
resolve_usage_ai_chat_model,
)
from litellm.router import Router
@ -1628,11 +1675,12 @@ class TestUsageAiChatTeamModelRouting:
),
):
await check_usage_ai_chat_model_access(user_api_key_dict=team_key, model=None)
routed_model: Final = await resolve_usage_ai_chat_model(user_api_key_dict=team_key, model=None)
events: Final = [
json.loads(raw.removeprefix("data: "))
async for raw in stream_usage_ai_chat(
messages=[{"role": "user", "content": "hi"}],
model=None,
model=routed_model,
is_admin=True,
user_api_key_dict=team_key,
)