mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(proxy): bill Ask AI calls to the caller and route team models like /chat/completions
Ask AI's router calls now carry the caller's key, user, and team the way /chat/completions does, so their spend is billed to that caller and the router's own team and tag filters pick the deployment. Admin UI sessions' pseudo team maps to no team, so admins keep cross-team routing and nothing is billed to it. Personal keys with no model picked also get the user-level model check, and the no-model 403 names the default model once.
This commit is contained in:
parent
3e2b631944
commit
67f701fdf4
2 changed files with 213 additions and 23 deletions
|
|
@ -6,9 +6,12 @@ usage/spend data by querying the aggregated daily activity endpoints.
|
|||
import json
|
||||
from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Mapping, Sequence
|
||||
from datetime import date
|
||||
from itertools import chain
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, Literal, NamedTuple, Protocol, TypeAlias, cast, overload
|
||||
|
||||
from fastapi import HTTPException
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
import litellm
|
||||
|
|
@ -470,7 +473,11 @@ def _resolve_model(model: str | None) -> str:
|
|||
_RouterRoute: TypeAlias = Literal["model_group", "deployment"]
|
||||
|
||||
|
||||
def _router_route(router: "Router", model: str, caller_is_proxy_admin: bool) -> _RouterRoute | None:
|
||||
def _router_route(
|
||||
router: "Router", model: str, team_id: str | None, caller_is_proxy_admin: bool
|
||||
) -> _RouterRoute | None:
|
||||
if team_id is not None and router.map_team_model(model, team_id) is not None:
|
||||
return "model_group"
|
||||
if router.is_recognized_model(model) or (caller_is_proxy_admin and model in router.team_public_model_names):
|
||||
return "model_group"
|
||||
if router.router_general_settings.pass_through_all_models:
|
||||
|
|
@ -480,6 +487,49 @@ def _router_route(router: "Router", model: str, caller_is_proxy_admin: bool) ->
|
|||
return "deployment" if model in router.deployment_names else None
|
||||
|
||||
|
||||
def _billed_caller(user_api_key_dict: "UserAPIKeyAuth") -> "UserAPIKeyAuth":
|
||||
from litellm.constants import UI_SESSION_TOKEN_TEAM_ID
|
||||
|
||||
if user_api_key_dict.team_id != UI_SESSION_TOKEN_TEAM_ID:
|
||||
return user_api_key_dict
|
||||
return user_api_key_dict.model_copy(update={"team_id": None})
|
||||
|
||||
|
||||
class _TaggedMetadata(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
tags: tuple[object, ...] | None = None
|
||||
|
||||
|
||||
class _CallerTagSources(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, from_attributes=True)
|
||||
|
||||
metadata: _TaggedMetadata | None = None
|
||||
team_metadata: _TaggedMetadata | None = None
|
||||
project_metadata: _TaggedMetadata | None = None
|
||||
|
||||
|
||||
def _caller_routing_tags(user_api_key_dict: "UserAPIKeyAuth") -> tuple[str, ...]:
|
||||
sources: Final = _CallerTagSources.model_validate(user_api_key_dict, from_attributes=True)
|
||||
tags: Final = chain.from_iterable(
|
||||
source.tags or () for source in (sources.metadata, sources.team_metadata, sources.project_metadata) if source
|
||||
)
|
||||
return tuple(dict.fromkeys(tag for tag in tags if isinstance(tag, str)))
|
||||
|
||||
|
||||
def _caller_metadata(caller: "UserAPIKeyAuth") -> Mapping[str, object]:
|
||||
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup
|
||||
|
||||
billing_metadata: Final = cast( # cast-ok: the shared helper returns the request dict untyped
|
||||
"Mapping[str, object]",
|
||||
LiteLLMProxyRequestSetup.add_user_api_key_auth_to_request_metadata( # pyright: ignore[reportUnknownMemberType] # the shared billing helper takes and returns a bare dict
|
||||
data={"metadata": {}}, user_api_key_dict=caller, _metadata_variable_name="metadata"
|
||||
)["metadata"],
|
||||
)
|
||||
caller_tags: Final = _caller_routing_tags(caller)
|
||||
return MappingProxyType({**billing_metadata, **({"tags": list(caller_tags)} if caller_tags else {})})
|
||||
|
||||
|
||||
async def _acompletion(
|
||||
model: str,
|
||||
messages: list[Mapping[str, object]], # mutable-ok: callers pass the request message list unchanged
|
||||
|
|
@ -488,10 +538,7 @@ async def _acompletion(
|
|||
user_api_key_dict: "UserAPIKeyAuth | None" = None,
|
||||
) -> ModelResponse | CustomStreamWrapper:
|
||||
from litellm.proxy.auth.auth_checks import key_model_aliases_for_auth_check
|
||||
from litellm.proxy.litellm_pre_call_utils import (
|
||||
resolve_team_model_alias,
|
||||
sanitized_user_api_key_auth,
|
||||
)
|
||||
from litellm.proxy.litellm_pre_call_utils import resolve_team_model_alias
|
||||
from litellm.proxy.proxy_server import llm_router
|
||||
from litellm.router_utils.common_utils import is_proxy_admin_request
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
|
|
@ -508,11 +555,17 @@ async def _acompletion(
|
|||
resolved_model: Final = (
|
||||
model_aliases.get(team_aliased_model, team_aliased_model) if model_aliases is not None else team_aliased_model
|
||||
)
|
||||
router_metadata: Final = (
|
||||
{"user_api_key_auth": sanitized_user_api_key_auth(user_api_key_dict)} if user_api_key_dict is not None else {}
|
||||
caller: Final = _billed_caller(user_api_key_dict) if user_api_key_dict is not None else None
|
||||
caller_metadata: Final[Mapping[str, object]] = (
|
||||
_caller_metadata(caller) if caller is not None else MappingProxyType({})
|
||||
)
|
||||
route: Final = (
|
||||
_router_route(llm_router, resolved_model, is_proxy_admin_request({"metadata": router_metadata}))
|
||||
_router_route(
|
||||
llm_router,
|
||||
resolved_model,
|
||||
caller.team_id if caller is not None else None,
|
||||
is_proxy_admin_request({"metadata": caller_metadata}),
|
||||
)
|
||||
if llm_router is not None
|
||||
else None
|
||||
)
|
||||
|
|
@ -526,7 +579,7 @@ async def _acompletion(
|
|||
temperature=USAGE_AI_TEMPERATURE,
|
||||
drop_params=True,
|
||||
allowed_openai_params=list(USAGE_AI_REQUIRED_PARAMS),
|
||||
metadata=router_metadata,
|
||||
metadata=dict(caller_metadata),
|
||||
specific_deployment=route == "deployment",
|
||||
)
|
||||
return await llm_router.acompletion( # pyright: ignore[reportUnknownMemberType] # Router.acompletion overloads carry untyped **kwargs
|
||||
|
|
@ -536,7 +589,7 @@ async def _acompletion(
|
|||
temperature=USAGE_AI_TEMPERATURE,
|
||||
drop_params=True,
|
||||
allowed_openai_params=list(USAGE_AI_REQUIRED_PARAMS),
|
||||
metadata=router_metadata,
|
||||
metadata=dict(caller_metadata),
|
||||
specific_deployment=route == "deployment",
|
||||
)
|
||||
return await litellm.acompletion(
|
||||
|
|
@ -547,15 +600,24 @@ async def _acompletion(
|
|||
temperature=USAGE_AI_TEMPERATURE,
|
||||
drop_params=True,
|
||||
allowed_openai_params=list(USAGE_AI_REQUIRED_PARAMS),
|
||||
metadata=dict(caller_metadata),
|
||||
)
|
||||
|
||||
|
||||
async def check_usage_ai_chat_model_access(user_api_key_dict: "UserAPIKeyAuth", model: str | None) -> None:
|
||||
from litellm.proxy._types import ProxyException
|
||||
from litellm.proxy.auth.auth_checks import can_key_call_resolved_model
|
||||
from litellm.proxy.auth.auth_checks import (
|
||||
_personal_key_owner, # pyright: ignore[reportPrivateUsage] # the same owner lookup the resolved-model check uses
|
||||
can_key_call_resolved_model,
|
||||
can_user_call_model,
|
||||
key_model_aliases_for_auth_check,
|
||||
)
|
||||
from litellm.proxy.proxy_server import (
|
||||
llm_model_list, # pyright: ignore[reportUnknownVariableType] # proxy_server.llm_model_list is an untyped module global
|
||||
llm_router,
|
||||
prisma_client,
|
||||
proxy_logging_obj,
|
||||
user_api_key_cache,
|
||||
)
|
||||
|
||||
resolved_model: Final = _resolve_model(model)
|
||||
|
|
@ -566,14 +628,23 @@ async def check_usage_ai_chat_model_access(user_api_key_dict: "UserAPIKeyAuth",
|
|||
valid_token=user_api_key_dict,
|
||||
llm_router=llm_router,
|
||||
)
|
||||
if user_api_key_dict.team_id is None:
|
||||
await can_user_call_model(
|
||||
model=resolved_model,
|
||||
llm_router=llm_router,
|
||||
user_object=await _personal_key_owner(
|
||||
valid_token=user_api_key_dict,
|
||||
prisma_client=prisma_client,
|
||||
user_api_key_cache=user_api_key_cache,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
),
|
||||
key_model_aliases=key_model_aliases_for_auth_check(user_api_key_dict),
|
||||
)
|
||||
except ProxyException as denial:
|
||||
if (model or "").strip():
|
||||
raise
|
||||
raise ProxyException(
|
||||
message=(
|
||||
f"Ask AI uses {resolved_model} when no model is picked, and this key cannot call it. "
|
||||
f"Pick a model this key can call. {denial.message}"
|
||||
),
|
||||
message=f"No model was picked, so Ask AI used its default. {denial.message} Pick a model you can call.",
|
||||
type=denial.type,
|
||||
param="model",
|
||||
code=denial.code,
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ import pytest
|
|||
import respx
|
||||
from fastapi import HTTPException
|
||||
|
||||
from litellm.proxy._types import LitellmUserRoles
|
||||
from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import (
|
||||
TOOL_HANDLERS,
|
||||
TOOLS_ADMIN,
|
||||
|
|
@ -1259,6 +1260,43 @@ class TestUsageAiChatModelAccess:
|
|||
|
||||
assert exc_info.value.code == "403"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(("model", "denied"), [(None, True), ("shared-model", False)])
|
||||
async def test_default_model_gets_the_user_model_check_a_picked_model_gets(self, model: str | None, denied: bool):
|
||||
from litellm.proxy._types import LiteLLM_UserTable, ProxyException, UserAPIKeyAuth
|
||||
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
|
||||
from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import (
|
||||
check_usage_ai_chat_model_access,
|
||||
)
|
||||
|
||||
user_cache: Final = UserApiKeyCache()
|
||||
await user_cache.async_set_cache(
|
||||
key="u1", value=LiteLLM_UserTable(user_id="u1", models=["shared-model"]), model_type=LiteLLM_UserTable
|
||||
)
|
||||
router: Final = self._router()
|
||||
with (
|
||||
patch( # test-quality-ok: check reads the proxy_server.llm_router module global; no injection seam
|
||||
"litellm.proxy.proxy_server.llm_router", router
|
||||
),
|
||||
patch( # test-quality-ok: check reads the proxy_server.llm_model_list module global; no injection seam
|
||||
"litellm.proxy.proxy_server.llm_model_list", None
|
||||
),
|
||||
patch( # test-quality-ok: check reads the proxy_server.prisma_client module global; no injection seam
|
||||
"litellm.proxy.proxy_server.prisma_client", None
|
||||
),
|
||||
patch( # test-quality-ok: check reads the proxy_server.user_api_key_cache module global; no injection seam
|
||||
"litellm.proxy.proxy_server.user_api_key_cache", user_cache
|
||||
),
|
||||
):
|
||||
personal_key: Final = UserAPIKeyAuth(models=[], user_id="u1", api_key="sk-test")
|
||||
if denied:
|
||||
with pytest.raises(ProxyException) as exc_info:
|
||||
await check_usage_ai_chat_model_access(user_api_key_dict=personal_key, model=model)
|
||||
assert exc_info.value.code == "403"
|
||||
assert "No model was picked" in exc_info.value.message
|
||||
else:
|
||||
assert await check_usage_ai_chat_model_access(user_api_key_dict=personal_key, model=model) is None
|
||||
|
||||
|
||||
class TestUsageAiChatTeamModelRouting:
|
||||
@staticmethod
|
||||
|
|
@ -1301,22 +1339,105 @@ class TestUsageAiChatTeamModelRouting:
|
|||
assert result.choices[0].message.content == "from the team deployment"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_team_key_ask_ai_spend_is_not_attributed_to_the_team(self, monkeypatch: pytest.MonkeyPatch):
|
||||
@pytest.mark.parametrize("user_role", [LitellmUserRoles.INTERNAL_USER, LitellmUserRoles.PROXY_ADMIN])
|
||||
async def test_team_key_reaches_its_own_team_deployment_when_teams_share_a_public_name(
|
||||
self, user_role: LitellmUserRoles
|
||||
):
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import (
|
||||
_acompletion,
|
||||
)
|
||||
from litellm.router import Router
|
||||
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": f"model_name_{team_id}_internal",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-4o",
|
||||
"api_key": "sk-dummy",
|
||||
"mock_response": f"from the {team_id} deployment",
|
||||
},
|
||||
"model_info": {"team_id": team_id, "team_public_model_name": "shared-team-model"},
|
||||
}
|
||||
for team_id in ("team-a", "team-b")
|
||||
]
|
||||
)
|
||||
|
||||
with patch( # test-quality-ok: _acompletion reads the proxy_server.llm_router module global; no injection seam
|
||||
"litellm.proxy.proxy_server.llm_router", router
|
||||
):
|
||||
result: Final = await _acompletion(
|
||||
model="shared-team-model",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
user_api_key_dict=UserAPIKeyAuth(user_role=user_role, team_id="team-b", api_key="sk-test"),
|
||||
)
|
||||
|
||||
assert result.choices[0].message.content == "from the team-b deployment"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_key_and_team_tags_pick_the_tagged_deployment(self):
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import (
|
||||
_acompletion,
|
||||
)
|
||||
from litellm.router import Router
|
||||
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "tagged-model",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-4o",
|
||||
"api_key": "sk-dummy",
|
||||
"mock_response": f"from the {tag} deployment",
|
||||
"tags": [tag],
|
||||
},
|
||||
}
|
||||
for tag in ("team-tag", "default")
|
||||
],
|
||||
enable_tag_filtering=True,
|
||||
)
|
||||
|
||||
with patch( # test-quality-ok: _acompletion reads the proxy_server.llm_router module global; no injection seam
|
||||
"litellm.proxy.proxy_server.llm_router", router
|
||||
):
|
||||
result: Final = await _acompletion(
|
||||
model="tagged-model",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
user_api_key_dict=UserAPIKeyAuth(
|
||||
team_id="team-a", team_metadata={"tags": ["team-tag"]}, metadata={}, api_key="sk-test"
|
||||
),
|
||||
)
|
||||
|
||||
assert result.choices[0].message.content == "from the team-tag deployment"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(("team_id", "billed_team_id"), [("team-a", "team-a"), ("litellm-dashboard", None)])
|
||||
async def test_ask_ai_spend_is_billed_to_the_calling_key_user_and_team(
|
||||
self, monkeypatch: pytest.MonkeyPatch, team_id: str, billed_team_id: str | None
|
||||
):
|
||||
import asyncio
|
||||
|
||||
import litellm
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import (
|
||||
_acompletion,
|
||||
)
|
||||
|
||||
logged: Final[asyncio.Queue[object]] = asyncio.Queue()
|
||||
logged: Final[asyncio.Queue[tuple[object, object, object]]] = asyncio.Queue()
|
||||
|
||||
class SpendCapture(CustomLogger):
|
||||
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
|
||||
await logged.put(kwargs["standard_logging_object"]["metadata"]["user_api_key_team_id"])
|
||||
metadata = kwargs["standard_logging_object"]["metadata"]
|
||||
await logged.put(
|
||||
(metadata["user_api_key_hash"], metadata["user_api_key_user_id"], metadata["user_api_key_team_id"])
|
||||
)
|
||||
|
||||
caller: Final = UserAPIKeyAuth(
|
||||
user_role=LitellmUserRoles.PROXY_ADMIN, team_id=team_id, user_id="u1", api_key="sk-test"
|
||||
)
|
||||
monkeypatch.setattr(litellm, "callbacks", [SpendCapture()])
|
||||
with patch( # test-quality-ok: _acompletion reads the proxy_server.llm_router module global; no injection seam
|
||||
"litellm.proxy.proxy_server.llm_router", self._team_router()
|
||||
|
|
@ -1324,12 +1445,10 @@ class TestUsageAiChatTeamModelRouting:
|
|||
await _acompletion(
|
||||
model="my-team-model",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
user_api_key_dict=UserAPIKeyAuth(
|
||||
user_role=LitellmUserRoles.PROXY_ADMIN, team_id="team-a", api_key="sk-test"
|
||||
),
|
||||
user_api_key_dict=caller,
|
||||
)
|
||||
|
||||
assert await asyncio.wait_for(logged.get(), timeout=10) is None
|
||||
assert await asyncio.wait_for(logged.get(), timeout=10) == (caller.api_key, "u1", billed_team_id)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_team_public_model_name_without_team_id_uses_litellm(self):
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue