From d3d5b9f37d0c4d14be64c62ce9f2a9f93aaa501f Mon Sep 17 00:00:00 2001 From: Krish Soni Date: Sun, 13 Sep 2026 17:17:50 +0530 Subject: [PATCH] refactor: remove soft_budget field from user budget interface documentation --- .../usage_endpoints/ai_usage_chat.py | 72 ++++++-- .../usage_endpoints/endpoints.py | 1 + .../usage_endpoints/test_ai_usage_chat.py | 161 +++++++++++------- ui/litellm-dashboard/src/lib/http/schema.d.ts | 2 - 4 files changed, 157 insertions(+), 79 deletions(-) diff --git a/litellm/proxy/management_endpoints/usage_endpoints/ai_usage_chat.py b/litellm/proxy/management_endpoints/usage_endpoints/ai_usage_chat.py index de0d90e0f20..4773d78d878 100644 --- a/litellm/proxy/management_endpoints/usage_endpoints/ai_usage_chat.py +++ b/litellm/proxy/management_endpoints/usage_endpoints/ai_usage_chat.py @@ -13,6 +13,9 @@ from typing_extensions import ReadOnly, TypedDict import litellm from litellm._logging import verbose_proxy_logger from litellm.constants import DEFAULT_COMPETITOR_DISCOVERY_MODEL +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup +from litellm.router import Router from litellm.types.proxy.management_endpoints.common_daily_activity import ( SpendAnalyticsPaginatedResponse, ) @@ -495,13 +498,14 @@ async def _process_tool_call( chat_messages: list[Mapping[str, object]], user_id: str | None, is_admin: bool, + tool_handlers: Mapping[str, ToolHandler], ) -> AsyncIterator[str]: """Execute a single tool call, yielding SSE events for status.""" fn_name: Final[str] = tc.function.name fn_args: Final[Mapping[str, str]] = json.loads(tc.function.arguments) allowed_names: Final = {t["function"]["name"] for t in get_tools_for_role(is_admin)} - handler: Final = TOOL_HANDLERS.get(fn_name) + handler: Final = tool_handlers.get(fn_name) if fn_name not in allowed_names or not handler: chat_messages.append( @@ -532,7 +536,39 @@ async def _process_tool_call( chat_messages.append({"role": "tool", "tool_call_id": tc.id, "content": tool_result}) -def _resolve_completion_fn(model: str) -> Callable[..., Awaitable[ModelResponse | CustomStreamWrapper]]: +class CompletionDeps(NamedTuple): + """Where to send a chat completion: the proxy's Router when it knows the model, else bare litellm.""" + + router: Router | None + acompletion: Callable[..., Awaitable[ModelResponse | CustomStreamWrapper]] + + +def _default_completion_deps() -> CompletionDeps: + from litellm.proxy.proxy_server import llm_router + + return CompletionDeps(router=llm_router, acompletion=litellm.acompletion) + + +def _call_metadata(user_api_key_dict: UserAPIKeyAuth | None) -> Mapping[str, object]: + """Stamp the caller's key/user/team identity onto the completion call. + + Without it, spend from these tool-planning and final-response completions is + never attributed to the calling key/user/team, so it silently doesn't count + against anyone's budget regardless of which model it is routed to. + """ + if user_api_key_dict is None: + return {} + stamped: Final = LiteLLMProxyRequestSetup.add_user_api_key_auth_to_request_metadata( # pyright: ignore[reportUnknownVariableType, reportUnknownMemberType] # untyped dict[Unknown, Unknown] signature upstream + data={"metadata": {}}, + user_api_key_dict=user_api_key_dict, + _metadata_variable_name="metadata", + ) + return cast(Mapping[str, object], stamped["metadata"]) + + +def _resolve_completion_fn( + model: str, deps: CompletionDeps +) -> Callable[..., Awaitable[ModelResponse | CustomStreamWrapper]]: """Pick the Router when it knows the model, else a direct litellm call. The "Ask AI" dropdown is populated from the proxy's model_list, so `model` is @@ -543,23 +579,27 @@ def _resolve_completion_fn(model: str) -> Callable[..., Awaitable[ModelResponse with no Router and for strings the Router does not recognise, such as a genuine provider-prefixed model or an unregistered DEFAULT_COMPETITOR_DISCOVERY_MODEL. """ - from litellm.proxy.proxy_server import llm_router - - if llm_router is not None and llm_router.get_model_list(model_name=model): - return llm_router.acompletion # pyright: ignore[reportUnknownVariableType] # untyped upstream signature - return litellm.acompletion # pyright: ignore[reportUnknownVariableType] # untyped upstream signature + if deps.router is not None and deps.router.get_model_list(model_name=model): + return deps.router.acompletion # pyright: ignore[reportUnknownVariableType] # untyped upstream signature + return deps.acompletion -async def _stream_final_response(model: str, chat_messages: list[dict[str, Any]]) -> AsyncIterator[str]: +async def _stream_final_response( + model: str, + chat_messages: list[dict[str, Any]], + deps: CompletionDeps, + metadata: Mapping[str, object], +) -> AsyncIterator[str]: """Stream the final LLM response after tool results are appended.""" yield _sse({"type": "status", "message": "Analyzing results..."}) - acompletion = _resolve_completion_fn(model) + acompletion = _resolve_completion_fn(model, deps) response = await acompletion( model=model, messages=chat_messages, stream=True, temperature=USAGE_AI_TEMPERATURE, + metadata=metadata, ) async for chunk in response: delta = chunk.choices[0].delta.content @@ -572,9 +612,16 @@ async def stream_usage_ai_chat( model: str | None = None, user_id: str | None = None, is_admin: bool = False, + *, + deps: CompletionDeps | None = None, + tool_handlers: Mapping[str, ToolHandler] | None = None, + user_api_key_dict: UserAPIKeyAuth | None = None, ) -> AsyncGenerator[str, None]: """Stream SSE events: status → tool_call → chunk → done.""" + resolved_deps: Final = deps if deps is not None else _default_completion_deps() + resolved_handlers: Final = tool_handlers if tool_handlers is not None else TOOL_HANDLERS resolved_model: Final = (model or "").strip() or DEFAULT_COMPETITOR_DISCOVERY_MODEL + metadata: Final = _call_metadata(user_api_key_dict) truncated: Final = messages[-MAX_CHAT_MESSAGES:] if len(messages) > MAX_CHAT_MESSAGES else messages chat_messages: Final[list[Mapping[str, object]]] = [ {"role": "system", "content": _build_system_prompt(is_admin)}, @@ -584,12 +631,13 @@ async def stream_usage_ai_chat( try: yield _sse({"type": "status", "message": "Thinking..."}) tools = get_tools_for_role(is_admin) - acompletion = _resolve_completion_fn(resolved_model) + acompletion = _resolve_completion_fn(resolved_model, resolved_deps) response = await acompletion( model=resolved_model, messages=chat_messages, tools=tools, temperature=USAGE_AI_TEMPERATURE, + metadata=metadata, ) choice: Final = response.choices[0] @@ -601,9 +649,9 @@ async def stream_usage_ai_chat( chat_messages.append(choice.message.model_dump()) for tc in choice.message.tool_calls: - async for event in _process_tool_call(tc, chat_messages, user_id, is_admin): + async for event in _process_tool_call(tc, chat_messages, user_id, is_admin, resolved_handlers): yield event - async for event in _stream_final_response(resolved_model, chat_messages): + async for event in _stream_final_response(resolved_model, chat_messages, resolved_deps, metadata): yield event yield _sse({"type": "done"}) diff --git a/litellm/proxy/management_endpoints/usage_endpoints/endpoints.py b/litellm/proxy/management_endpoints/usage_endpoints/endpoints.py index d1e92d0c7e4..a95353e41fa 100644 --- a/litellm/proxy/management_endpoints/usage_endpoints/endpoints.py +++ b/litellm/proxy/management_endpoints/usage_endpoints/endpoints.py @@ -67,6 +67,7 @@ async def usage_ai_chat( model=data.model, user_id=user_id, is_admin=is_admin, + user_api_key_dict=user_api_key_dict, ), ping_interval_seconds=litellm.sse_keepalive_ping_interval_seconds, ping_chunk=SSE_COMMENT_PING, diff --git a/tests/test_litellm/proxy/management_endpoints/usage_endpoints/test_ai_usage_chat.py b/tests/test_litellm/proxy/management_endpoints/usage_endpoints/test_ai_usage_chat.py index ed71bac9825..f2390eeda11 100644 --- a/tests/test_litellm/proxy/management_endpoints/usage_endpoints/test_ai_usage_chat.py +++ b/tests/test_litellm/proxy/management_endpoints/usage_endpoints/test_ai_usage_chat.py @@ -8,16 +8,15 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import ( - TOOL_HANDLERS, TOOLS_ADMIN, TOOLS_BASE, + CompletionDeps, _build_system_prompt, _summarise_entity_data, _summarise_usage_data, stream_usage_ai_chat, ) - SAMPLE_AGGREGATED_RESPONSE = { "results": [ { @@ -149,22 +148,15 @@ async def _collect_events(**kwargs): return [json.loads(event.removeprefix("data: ").strip()) async for event in stream_usage_ai_chat(**kwargs)] -def _patched_usage_tool(): - """Replace the get_usage_data handler so no DB query is attempted. - - TOOL_HANDLERS captures `_fetch_usage_data` at import time, so patching the - module attribute alone leaves the real (DB-backed) fetcher wired up. - """ - return patch.dict( - "litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.TOOL_HANDLERS", - { - "get_usage_data": { - "fetch": AsyncMock(return_value=SAMPLE_AGGREGATED_RESPONSE), - "summarise": _summarise_usage_data, - "label": "global usage data", - } - }, - ) +def _usage_tool_handlers(): + """Tool handler override so no DB query is attempted, injected via `tool_handlers=`.""" + return { + "get_usage_data": { + "fetch": AsyncMock(return_value=SAMPLE_AGGREGATED_RESPONSE), + "summarise": _summarise_usage_data, + "label": "global usage data", + } + } class TestToolSchemas: @@ -482,25 +474,21 @@ class TestRouterResolution: mock_router.acompletion = AsyncMock( side_effect=[_make_tool_call_response(), _make_stream("Total spend is $50.25")] ) + mock_acompletion = AsyncMock() - with ( - patch("litellm.proxy.proxy_server.llm_router", mock_router), - patch("litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.litellm") as mock_litellm, - _patched_usage_tool(), - ): - mock_litellm.acompletion = AsyncMock() - - events = await _collect_events( - messages=[{"role": "user", "content": "What is my total spend?"}], - model="my-gpt4", - user_id="user-123", - is_admin=True, - ) + events = await _collect_events( + messages=[{"role": "user", "content": "What is my total spend?"}], + model="my-gpt4", + user_id="user-123", + is_admin=True, + deps=CompletionDeps(router=mock_router, acompletion=mock_acompletion), + tool_handlers=_usage_tool_handlers(), + ) assert [e for e in events if e["type"] == "error"] == [] assert [e["status"] for e in events if e["type"] == "tool_call"] == ["running", "complete"] assert [e["content"] for e in events if e["type"] == "chunk"] == ["Total spend is $50.25"] - mock_litellm.acompletion.assert_not_called() + mock_acompletion.assert_not_called() mock_router.get_model_list.assert_called_with(model_name="my-gpt4") assert mock_router.acompletion.await_count == 2 @@ -517,48 +505,90 @@ class TestRouterResolution: mock_router = MagicMock() mock_router.get_model_list.return_value = [] mock_router.acompletion = AsyncMock() + mock_acompletion = AsyncMock(side_effect=[_make_tool_call_response(), _make_stream("Spend summary")]) - with ( - patch("litellm.proxy.proxy_server.llm_router", mock_router), - patch("litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.litellm") as mock_litellm, - _patched_usage_tool(), - ): - mock_litellm.acompletion = AsyncMock( - side_effect=[_make_tool_call_response(), _make_stream("Spend summary")] - ) - - events = await _collect_events( - messages=[{"role": "user", "content": "What is my total spend?"}], - model="openai/gpt-4o-mini", - user_id="user-123", - is_admin=True, - ) + events = await _collect_events( + messages=[{"role": "user", "content": "What is my total spend?"}], + model="openai/gpt-4o-mini", + user_id="user-123", + is_admin=True, + deps=CompletionDeps(router=mock_router, acompletion=mock_acompletion), + tool_handlers=_usage_tool_handlers(), + ) assert [e for e in events if e["type"] == "error"] == [] mock_router.acompletion.assert_not_called() - assert mock_litellm.acompletion.await_count == 2 - assert mock_litellm.acompletion.await_args_list[0].kwargs["model"] == "openai/gpt-4o-mini" + assert mock_acompletion.await_count == 2 + assert mock_acompletion.await_args_list[0].kwargs["model"] == "openai/gpt-4o-mini" @pytest.mark.asyncio async def test_no_router_configured_uses_litellm(self): - with ( - patch("litellm.proxy.proxy_server.llm_router", None), - patch("litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.litellm") as mock_litellm, - _patched_usage_tool(), - ): - mock_litellm.acompletion = AsyncMock( - side_effect=[_make_tool_call_response(), _make_stream("Spend summary")] - ) + mock_acompletion = AsyncMock(side_effect=[_make_tool_call_response(), _make_stream("Spend summary")]) - events = await _collect_events( - messages=[{"role": "user", "content": "What is my total spend?"}], - model="gpt-4o-mini", - user_id="user-123", - is_admin=True, - ) + events = await _collect_events( + messages=[{"role": "user", "content": "What is my total spend?"}], + model="gpt-4o-mini", + user_id="user-123", + is_admin=True, + deps=CompletionDeps(router=None, acompletion=mock_acompletion), + tool_handlers=_usage_tool_handlers(), + ) assert [e for e in events if e["type"] == "error"] == [] - assert mock_litellm.acompletion.await_count == 2 + assert mock_acompletion.await_count == 2 + + +class TestUsageAiChatSpendAttribution: + """ + Regression: completions issued by the Usage dashboard's "Ask AI" chat must + carry the caller's key/user/team identity in `metadata`, the same as every + other proxied completion, so spend gets attributed and counts against the + caller's budget. Previously these completions carried no metadata at all, + so their cost was untracked regardless of which model handled them. + """ + + @pytest.mark.asyncio + async def test_both_completions_carry_caller_identity_in_metadata(self): + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + + mock_acompletion = AsyncMock(side_effect=[_make_tool_call_response(), _make_stream("Total spend is $50.25")]) + + await _collect_events( + messages=[{"role": "user", "content": "What is my total spend?"}], + model="gpt-4o-mini", + user_id="user-123", + is_admin=True, + deps=CompletionDeps(router=None, acompletion=mock_acompletion), + tool_handlers=_usage_tool_handlers(), + user_api_key_dict=UserAPIKeyAuth( + api_key="hashed-key", + user_id="user-123", + team_id="team-456", + user_role=LitellmUserRoles.INTERNAL_USER, + ), + ) + + assert mock_acompletion.await_count == 2 + for call in mock_acompletion.await_args_list: + assert call.kwargs["metadata"]["user_api_key_user_id"] == "user-123" + assert call.kwargs["metadata"]["user_api_key_team_id"] == "team-456" + assert call.kwargs["metadata"]["user_api_key"] == "hashed-key" + + @pytest.mark.asyncio + async def test_metadata_is_empty_without_a_caller_identity(self): + mock_acompletion = AsyncMock(side_effect=[_make_tool_call_response(), _make_stream("Spend summary")]) + + await _collect_events( + messages=[{"role": "user", "content": "What is my total spend?"}], + model="gpt-4o-mini", + user_id="user-123", + is_admin=True, + deps=CompletionDeps(router=None, acompletion=mock_acompletion), + tool_handlers=_usage_tool_handlers(), + ) + + for call in mock_acompletion.await_args_list: + assert call.kwargs["metadata"] == {} class TestUsageAiChatServiceAccountGuard: @@ -622,8 +652,9 @@ class TestUsageAiChatKeepalive: async def _collect_endpoint_body(self, monkeypatch, interval, delay=0.3) -> tuple[list[bytes], dict]: import asyncio - import litellm from fastapi.responses import StreamingResponse + + import litellm from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.management_endpoints.usage_endpoints.endpoints import ( ChatMessage, diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 7eadaa6c991..839aa52fa84 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -16781,7 +16781,6 @@ export interface paths { * - permissions: Optional[dict] - [Not Implemented Yet] User-specific permissions, eg. turning off pii masking. * - metadata: Optional[dict] - Metadata for user, store information for user. Example metadata = {"team": "core-infra", "app": "app2", "email": "ishaan@berri.ai" } * - max_parallel_requests: Optional[int] - Rate limit a user based on the number of parallel requests. Raises 429 error, if user's parallel requests > x. - * - soft_budget: Optional[float] - Get alerts when user crosses given budget, doesn't block requests. * - model_max_budget: Optional[dict] - Model-specific max budget for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-budgets-to-keys) * - budget_fallbacks: Optional[Dict[str, List[str]]] - Per-model fallback chain tried in order when that model's own `model_max_budget` is exceeded, e.g. {"gpt-4o": ["gpt-4o-mini"]}. * - model_rpm_limit: Optional[float] - Model-specific rpm limit for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-limits-to-keys) @@ -16887,7 +16886,6 @@ export interface paths { * - permissions: Optional[dict] - [Not Implemented Yet] User-specific permissions, eg. turning off pii masking. * - metadata: Optional[dict] - Metadata for user, store information for user. Example metadata = {"team": "core-infra", "app": "app2", "email": "ishaan@berri.ai" } * - max_parallel_requests: Optional[int] - Rate limit a user based on the number of parallel requests. Raises 429 error, if user's parallel requests > x. - * - soft_budget: Optional[float] - Get alerts when user crosses given budget, doesn't block requests. * - model_max_budget: Optional[dict] - Model-specific max budget for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-budgets-to-keys) * - budget_fallbacks: Optional[Dict[str, List[str]]] - Per-model fallback chain tried in order when that model's own `model_max_budget` is exceeded, e.g. {"gpt-4o": ["gpt-4o-mini"]}. * - model_rpm_limit: Optional[float] - Model-specific rpm limit for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-limits-to-keys)