fix(proxy): bill no caller for usage Ask AI calls, as before this change

Billing the calling key used up its budget and TPM on a route that checks
neither, so the same key's /chat/completions could start answering 429
This commit is contained in:
mateo-berri 2026-10-20 10:02:52 -07:00
parent 5d565bab94
commit a011b3670b
2 changed files with 12 additions and 72 deletions

View file

@ -16,7 +16,7 @@ from typing_extensions import ReadOnly, TypedDict
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.constants import DEFAULT_COMPETITOR_DISCOVERY_MODEL, UI_SESSION_TOKEN_TEAM_ID
from litellm.constants import DEFAULT_COMPETITOR_DISCOVERY_MODEL
from litellm.proxy._types import CommonProxyErrors
from litellm.types.proxy.management_endpoints.common_daily_activity import (
SpendAnalyticsPaginatedResponse,
@ -474,11 +474,7 @@ def _resolve_model(model: str | None) -> str:
_RouterRoute: TypeAlias = Literal["model_group", "deployment"]
def _router_route(
router: "Router", model: str, team_id: str | None, caller_is_proxy_admin: bool
) -> _RouterRoute | None:
if team_id is not None and router.map_team_model(model, team_id) is not None:
return "model_group"
def _router_route(router: "Router", model: str, caller_is_proxy_admin: bool) -> _RouterRoute | None:
if router.is_recognized_model(model) or (caller_is_proxy_admin and model in router.team_public_model_names):
return "model_group"
if router.router_general_settings.pass_through_all_models:
@ -515,24 +511,13 @@ def _caller_routing_tags(user_api_key_dict: "UserAPIKeyAuth") -> tuple[str, ...]
return tuple(dict.fromkeys(tag for tag in tags if isinstance(tag, str)))
def _caller_identity_metadata(user_api_key_dict: "UserAPIKeyAuth") -> Mapping[str, object]:
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup, sanitized_user_api_key_auth
if user_api_key_dict.team_id == UI_SESSION_TOKEN_TEAM_ID:
return MappingProxyType({"user_api_key_auth": sanitized_user_api_key_auth(user_api_key_dict)})
return cast( # cast-ok: the shared helper returns the request dict untyped
"Mapping[str, object]",
LiteLLMProxyRequestSetup.add_user_api_key_auth_to_request_metadata( # pyright: ignore[reportUnknownMemberType] # the shared billing helper takes and returns a bare dict
data={"metadata": {}}, user_api_key_dict=user_api_key_dict, _metadata_variable_name="metadata"
)["metadata"],
)
def _caller_metadata(user_api_key_dict: "UserAPIKeyAuth") -> Mapping[str, object]:
from litellm.proxy.litellm_pre_call_utils import sanitized_user_api_key_auth
caller_tags: Final = _caller_routing_tags(user_api_key_dict)
return MappingProxyType(
{
**_caller_identity_metadata(user_api_key_dict),
"user_api_key_auth": sanitized_user_api_key_auth(user_api_key_dict),
**({"tags": list(caller_tags), "inherited_tags": list(caller_tags)} if caller_tags else {}),
}
)
@ -619,7 +604,7 @@ async def _acompletion(
router_settings: Mapping[str, object] | None = None,
) -> ModelResponse | CustomStreamWrapper:
from litellm.proxy.proxy_server import llm_router
from litellm.router_utils.common_utils import get_request_team_id, is_proxy_admin_request
from litellm.router_utils.common_utils import is_proxy_admin_request
from litellm.types.llms.openai import AllMessageValues
completion_messages: Final = cast( # cast-ok: acompletion accepts the same Mapping messages it already receives
@ -629,12 +614,7 @@ async def _acompletion(
_caller_metadata(user_api_key_dict) if user_api_key_dict is not None else MappingProxyType({})
)
route: Final = (
_router_route(
llm_router,
model,
get_request_team_id({"metadata": caller_metadata}),
is_proxy_admin_request({"metadata": caller_metadata}),
)
_router_route(llm_router, model, is_proxy_admin_request({"metadata": caller_metadata}))
if llm_router is not None
else None
)

View file

@ -12,7 +12,7 @@ import pytest
import respx
from fastapi import HTTPException
from litellm.proxy._types import LitellmUserRoles, hash_token
from litellm.proxy._types import LitellmUserRoles
from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import (
TOOL_HANDLERS,
TOOLS_ADMIN,
@ -1351,43 +1351,6 @@ class TestUsageAiChatTeamModelRouting:
assert result.choices[0].message.content == "from the team deployment"
@pytest.mark.asyncio
@pytest.mark.parametrize("user_role", [LitellmUserRoles.INTERNAL_USER, LitellmUserRoles.PROXY_ADMIN])
async def test_team_key_reaches_its_own_team_deployment_when_teams_share_a_public_name(
self, user_role: LitellmUserRoles
):
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import (
_acompletion,
)
from litellm.router import Router
router: Final = Router(
model_list=[
{
"model_name": f"model_name_{team_id}_internal",
"litellm_params": {
"model": "openai/gpt-4o",
"api_key": "sk-dummy",
"mock_response": f"from the {team_id} deployment",
},
"model_info": {"team_id": team_id, "team_public_model_name": "shared-team-model"},
}
for team_id in ("team-a", "team-b")
]
)
with patch( # test-quality-ok: _acompletion reads the proxy_server.llm_router module global; no injection seam
"litellm.proxy.proxy_server.llm_router", router
):
result: Final = await _acompletion(
model="shared-team-model",
messages=[{"role": "user", "content": "hi"}],
user_api_key_dict=UserAPIKeyAuth(user_role=user_role, team_id="team-b", api_key="sk-test"),
)
assert result.choices[0].message.content == "from the team-b deployment"
@pytest.mark.asyncio
@pytest.mark.parametrize("key_metadata", [{}, {"tags": "prod"}, {"tags": {"env": "prod"}}])
@pytest.mark.parametrize("tag_routing_prefix", ["", "route:"])
@ -1431,12 +1394,9 @@ class TestUsageAiChatTeamModelRouting:
assert result.choices[0].message.content == "from the team-tag deployment"
@pytest.mark.asyncio
@pytest.mark.parametrize(
("team_id", "billed_caller"),
[("team-a", (hash_token("sk-test"), "u1", "team-a")), ("litellm-dashboard", (None, None, None))],
)
async def test_ask_ai_bills_an_api_key_caller_and_never_an_admin_ui_session(
self, monkeypatch: pytest.MonkeyPatch, team_id: str, billed_caller: tuple[str | None, str | None, str | None]
@pytest.mark.parametrize("team_id", ["team-a", "litellm-dashboard"])
async def test_ask_ai_charges_no_key_user_or_team_spend_or_rate_limit(
self, monkeypatch: pytest.MonkeyPatch, team_id: str
):
import asyncio
@ -1469,7 +1429,7 @@ class TestUsageAiChatTeamModelRouting:
user_api_key_dict=caller,
)
assert await asyncio.wait_for(logged.get(), timeout=10) == billed_caller
assert await asyncio.wait_for(logged.get(), timeout=10) == (None, None, None)
@pytest.mark.asyncio
async def test_team_public_model_name_without_team_id_uses_litellm(self):