mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(proxy): bill no caller for usage Ask AI calls, as before this change
Billing the calling key used up its budget and TPM on a route that checks neither, so the same key's /chat/completions could start answering 429
This commit is contained in:
parent
5d565bab94
commit
a011b3670b
2 changed files with 12 additions and 72 deletions
|
|
@ -16,7 +16,7 @@ from typing_extensions import ReadOnly, TypedDict
|
|||
|
||||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.constants import DEFAULT_COMPETITOR_DISCOVERY_MODEL, UI_SESSION_TOKEN_TEAM_ID
|
||||
from litellm.constants import DEFAULT_COMPETITOR_DISCOVERY_MODEL
|
||||
from litellm.proxy._types import CommonProxyErrors
|
||||
from litellm.types.proxy.management_endpoints.common_daily_activity import (
|
||||
SpendAnalyticsPaginatedResponse,
|
||||
|
|
@ -474,11 +474,7 @@ def _resolve_model(model: str | None) -> str:
|
|||
_RouterRoute: TypeAlias = Literal["model_group", "deployment"]
|
||||
|
||||
|
||||
def _router_route(
|
||||
router: "Router", model: str, team_id: str | None, caller_is_proxy_admin: bool
|
||||
) -> _RouterRoute | None:
|
||||
if team_id is not None and router.map_team_model(model, team_id) is not None:
|
||||
return "model_group"
|
||||
def _router_route(router: "Router", model: str, caller_is_proxy_admin: bool) -> _RouterRoute | None:
|
||||
if router.is_recognized_model(model) or (caller_is_proxy_admin and model in router.team_public_model_names):
|
||||
return "model_group"
|
||||
if router.router_general_settings.pass_through_all_models:
|
||||
|
|
@ -515,24 +511,13 @@ def _caller_routing_tags(user_api_key_dict: "UserAPIKeyAuth") -> tuple[str, ...]
|
|||
return tuple(dict.fromkeys(tag for tag in tags if isinstance(tag, str)))
|
||||
|
||||
|
||||
def _caller_identity_metadata(user_api_key_dict: "UserAPIKeyAuth") -> Mapping[str, object]:
|
||||
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup, sanitized_user_api_key_auth
|
||||
|
||||
if user_api_key_dict.team_id == UI_SESSION_TOKEN_TEAM_ID:
|
||||
return MappingProxyType({"user_api_key_auth": sanitized_user_api_key_auth(user_api_key_dict)})
|
||||
return cast( # cast-ok: the shared helper returns the request dict untyped
|
||||
"Mapping[str, object]",
|
||||
LiteLLMProxyRequestSetup.add_user_api_key_auth_to_request_metadata( # pyright: ignore[reportUnknownMemberType] # the shared billing helper takes and returns a bare dict
|
||||
data={"metadata": {}}, user_api_key_dict=user_api_key_dict, _metadata_variable_name="metadata"
|
||||
)["metadata"],
|
||||
)
|
||||
|
||||
|
||||
def _caller_metadata(user_api_key_dict: "UserAPIKeyAuth") -> Mapping[str, object]:
|
||||
from litellm.proxy.litellm_pre_call_utils import sanitized_user_api_key_auth
|
||||
|
||||
caller_tags: Final = _caller_routing_tags(user_api_key_dict)
|
||||
return MappingProxyType(
|
||||
{
|
||||
**_caller_identity_metadata(user_api_key_dict),
|
||||
"user_api_key_auth": sanitized_user_api_key_auth(user_api_key_dict),
|
||||
**({"tags": list(caller_tags), "inherited_tags": list(caller_tags)} if caller_tags else {}),
|
||||
}
|
||||
)
|
||||
|
|
@ -619,7 +604,7 @@ async def _acompletion(
|
|||
router_settings: Mapping[str, object] | None = None,
|
||||
) -> ModelResponse | CustomStreamWrapper:
|
||||
from litellm.proxy.proxy_server import llm_router
|
||||
from litellm.router_utils.common_utils import get_request_team_id, is_proxy_admin_request
|
||||
from litellm.router_utils.common_utils import is_proxy_admin_request
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
|
||||
completion_messages: Final = cast( # cast-ok: acompletion accepts the same Mapping messages it already receives
|
||||
|
|
@ -629,12 +614,7 @@ async def _acompletion(
|
|||
_caller_metadata(user_api_key_dict) if user_api_key_dict is not None else MappingProxyType({})
|
||||
)
|
||||
route: Final = (
|
||||
_router_route(
|
||||
llm_router,
|
||||
model,
|
||||
get_request_team_id({"metadata": caller_metadata}),
|
||||
is_proxy_admin_request({"metadata": caller_metadata}),
|
||||
)
|
||||
_router_route(llm_router, model, is_proxy_admin_request({"metadata": caller_metadata}))
|
||||
if llm_router is not None
|
||||
else None
|
||||
)
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ import pytest
|
|||
import respx
|
||||
from fastapi import HTTPException
|
||||
|
||||
from litellm.proxy._types import LitellmUserRoles, hash_token
|
||||
from litellm.proxy._types import LitellmUserRoles
|
||||
from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import (
|
||||
TOOL_HANDLERS,
|
||||
TOOLS_ADMIN,
|
||||
|
|
@ -1351,43 +1351,6 @@ class TestUsageAiChatTeamModelRouting:
|
|||
|
||||
assert result.choices[0].message.content == "from the team deployment"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("user_role", [LitellmUserRoles.INTERNAL_USER, LitellmUserRoles.PROXY_ADMIN])
|
||||
async def test_team_key_reaches_its_own_team_deployment_when_teams_share_a_public_name(
|
||||
self, user_role: LitellmUserRoles
|
||||
):
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import (
|
||||
_acompletion,
|
||||
)
|
||||
from litellm.router import Router
|
||||
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": f"model_name_{team_id}_internal",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-4o",
|
||||
"api_key": "sk-dummy",
|
||||
"mock_response": f"from the {team_id} deployment",
|
||||
},
|
||||
"model_info": {"team_id": team_id, "team_public_model_name": "shared-team-model"},
|
||||
}
|
||||
for team_id in ("team-a", "team-b")
|
||||
]
|
||||
)
|
||||
|
||||
with patch( # test-quality-ok: _acompletion reads the proxy_server.llm_router module global; no injection seam
|
||||
"litellm.proxy.proxy_server.llm_router", router
|
||||
):
|
||||
result: Final = await _acompletion(
|
||||
model="shared-team-model",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
user_api_key_dict=UserAPIKeyAuth(user_role=user_role, team_id="team-b", api_key="sk-test"),
|
||||
)
|
||||
|
||||
assert result.choices[0].message.content == "from the team-b deployment"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("key_metadata", [{}, {"tags": "prod"}, {"tags": {"env": "prod"}}])
|
||||
@pytest.mark.parametrize("tag_routing_prefix", ["", "route:"])
|
||||
|
|
@ -1431,12 +1394,9 @@ class TestUsageAiChatTeamModelRouting:
|
|||
assert result.choices[0].message.content == "from the team-tag deployment"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("team_id", "billed_caller"),
|
||||
[("team-a", (hash_token("sk-test"), "u1", "team-a")), ("litellm-dashboard", (None, None, None))],
|
||||
)
|
||||
async def test_ask_ai_bills_an_api_key_caller_and_never_an_admin_ui_session(
|
||||
self, monkeypatch: pytest.MonkeyPatch, team_id: str, billed_caller: tuple[str | None, str | None, str | None]
|
||||
@pytest.mark.parametrize("team_id", ["team-a", "litellm-dashboard"])
|
||||
async def test_ask_ai_charges_no_key_user_or_team_spend_or_rate_limit(
|
||||
self, monkeypatch: pytest.MonkeyPatch, team_id: str
|
||||
):
|
||||
import asyncio
|
||||
|
||||
|
|
@ -1469,7 +1429,7 @@ class TestUsageAiChatTeamModelRouting:
|
|||
user_api_key_dict=caller,
|
||||
)
|
||||
|
||||
assert await asyncio.wait_for(logged.get(), timeout=10) == billed_caller
|
||||
assert await asyncio.wait_for(logged.get(), timeout=10) == (None, None, None)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_team_public_model_name_without_team_id_uses_litellm(self):
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue