refactor: remove soft_budget field from user budget interface documentation

This commit is contained in:
Krish Soni 2026-09-13 17:17:50 +05:30
parent 58b92fa7f8
commit d3d5b9f37d
4 changed files with 157 additions and 79 deletions

View file

@ -13,6 +13,9 @@ from typing_extensions import ReadOnly, TypedDict
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.constants import DEFAULT_COMPETITOR_DISCOVERY_MODEL
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup
from litellm.router import Router
from litellm.types.proxy.management_endpoints.common_daily_activity import (
SpendAnalyticsPaginatedResponse,
)
@ -495,13 +498,14 @@ async def _process_tool_call(
chat_messages: list[Mapping[str, object]],
user_id: str | None,
is_admin: bool,
tool_handlers: Mapping[str, ToolHandler],
) -> AsyncIterator[str]:
"""Execute a single tool call, yielding SSE events for status."""
fn_name: Final[str] = tc.function.name
fn_args: Final[Mapping[str, str]] = json.loads(tc.function.arguments)
allowed_names: Final = {t["function"]["name"] for t in get_tools_for_role(is_admin)}
handler: Final = TOOL_HANDLERS.get(fn_name)
handler: Final = tool_handlers.get(fn_name)
if fn_name not in allowed_names or not handler:
chat_messages.append(
@ -532,7 +536,39 @@ async def _process_tool_call(
chat_messages.append({"role": "tool", "tool_call_id": tc.id, "content": tool_result})
def _resolve_completion_fn(model: str) -> Callable[..., Awaitable[ModelResponse | CustomStreamWrapper]]:
class CompletionDeps(NamedTuple):
"""Where to send a chat completion: the proxy's Router when it knows the model, else bare litellm."""
router: Router | None
acompletion: Callable[..., Awaitable[ModelResponse | CustomStreamWrapper]]
def _default_completion_deps() -> CompletionDeps:
from litellm.proxy.proxy_server import llm_router
return CompletionDeps(router=llm_router, acompletion=litellm.acompletion)
def _call_metadata(user_api_key_dict: UserAPIKeyAuth | None) -> Mapping[str, object]:
"""Stamp the caller's key/user/team identity onto the completion call.
Without it, spend from these tool-planning and final-response completions is
never attributed to the calling key/user/team, so it silently doesn't count
against anyone's budget regardless of which model it is routed to.
"""
if user_api_key_dict is None:
return {}
stamped: Final = LiteLLMProxyRequestSetup.add_user_api_key_auth_to_request_metadata( # pyright: ignore[reportUnknownVariableType, reportUnknownMemberType] # untyped dict[Unknown, Unknown] signature upstream
data={"metadata": {}},
user_api_key_dict=user_api_key_dict,
_metadata_variable_name="metadata",
)
return cast(Mapping[str, object], stamped["metadata"])
def _resolve_completion_fn(
model: str, deps: CompletionDeps
) -> Callable[..., Awaitable[ModelResponse | CustomStreamWrapper]]:
"""Pick the Router when it knows the model, else a direct litellm call.
The "Ask AI" dropdown is populated from the proxy's model_list, so `model` is
@ -543,23 +579,27 @@ def _resolve_completion_fn(model: str) -> Callable[..., Awaitable[ModelResponse
with no Router and for strings the Router does not recognise, such as a genuine
provider-prefixed model or an unregistered DEFAULT_COMPETITOR_DISCOVERY_MODEL.
"""
from litellm.proxy.proxy_server import llm_router
if llm_router is not None and llm_router.get_model_list(model_name=model):
return llm_router.acompletion # pyright: ignore[reportUnknownVariableType] # untyped upstream signature
return litellm.acompletion # pyright: ignore[reportUnknownVariableType] # untyped upstream signature
if deps.router is not None and deps.router.get_model_list(model_name=model):
return deps.router.acompletion # pyright: ignore[reportUnknownVariableType] # untyped upstream signature
return deps.acompletion
async def _stream_final_response(model: str, chat_messages: list[dict[str, Any]]) -> AsyncIterator[str]:
async def _stream_final_response(
model: str,
chat_messages: list[dict[str, Any]],
deps: CompletionDeps,
metadata: Mapping[str, object],
) -> AsyncIterator[str]:
"""Stream the final LLM response after tool results are appended."""
yield _sse({"type": "status", "message": "Analyzing results..."})
acompletion = _resolve_completion_fn(model)
acompletion = _resolve_completion_fn(model, deps)
response = await acompletion(
model=model,
messages=chat_messages,
stream=True,
temperature=USAGE_AI_TEMPERATURE,
metadata=metadata,
)
async for chunk in response:
delta = chunk.choices[0].delta.content
@ -572,9 +612,16 @@ async def stream_usage_ai_chat(
model: str | None = None,
user_id: str | None = None,
is_admin: bool = False,
*,
deps: CompletionDeps | None = None,
tool_handlers: Mapping[str, ToolHandler] | None = None,
user_api_key_dict: UserAPIKeyAuth | None = None,
) -> AsyncGenerator[str, None]:
"""Stream SSE events: status → tool_call → chunk → done."""
resolved_deps: Final = deps if deps is not None else _default_completion_deps()
resolved_handlers: Final = tool_handlers if tool_handlers is not None else TOOL_HANDLERS
resolved_model: Final = (model or "").strip() or DEFAULT_COMPETITOR_DISCOVERY_MODEL
metadata: Final = _call_metadata(user_api_key_dict)
truncated: Final = messages[-MAX_CHAT_MESSAGES:] if len(messages) > MAX_CHAT_MESSAGES else messages
chat_messages: Final[list[Mapping[str, object]]] = [
{"role": "system", "content": _build_system_prompt(is_admin)},
@ -584,12 +631,13 @@ async def stream_usage_ai_chat(
try:
yield _sse({"type": "status", "message": "Thinking..."})
tools = get_tools_for_role(is_admin)
acompletion = _resolve_completion_fn(resolved_model)
acompletion = _resolve_completion_fn(resolved_model, resolved_deps)
response = await acompletion(
model=resolved_model,
messages=chat_messages,
tools=tools,
temperature=USAGE_AI_TEMPERATURE,
metadata=metadata,
)
choice: Final = response.choices[0]
@ -601,9 +649,9 @@ async def stream_usage_ai_chat(
chat_messages.append(choice.message.model_dump())
for tc in choice.message.tool_calls:
async for event in _process_tool_call(tc, chat_messages, user_id, is_admin):
async for event in _process_tool_call(tc, chat_messages, user_id, is_admin, resolved_handlers):
yield event
async for event in _stream_final_response(resolved_model, chat_messages):
async for event in _stream_final_response(resolved_model, chat_messages, resolved_deps, metadata):
yield event
yield _sse({"type": "done"})

View file

@ -67,6 +67,7 @@ async def usage_ai_chat(
model=data.model,
user_id=user_id,
is_admin=is_admin,
user_api_key_dict=user_api_key_dict,
),
ping_interval_seconds=litellm.sse_keepalive_ping_interval_seconds,
ping_chunk=SSE_COMMENT_PING,

View file

@ -8,16 +8,15 @@ from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import (
TOOL_HANDLERS,
TOOLS_ADMIN,
TOOLS_BASE,
CompletionDeps,
_build_system_prompt,
_summarise_entity_data,
_summarise_usage_data,
stream_usage_ai_chat,
)
SAMPLE_AGGREGATED_RESPONSE = {
"results": [
{
@ -149,22 +148,15 @@ async def _collect_events(**kwargs):
return [json.loads(event.removeprefix("data: ").strip()) async for event in stream_usage_ai_chat(**kwargs)]
def _patched_usage_tool():
"""Replace the get_usage_data handler so no DB query is attempted.
TOOL_HANDLERS captures `_fetch_usage_data` at import time, so patching the
module attribute alone leaves the real (DB-backed) fetcher wired up.
"""
return patch.dict(
"litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.TOOL_HANDLERS",
{
"get_usage_data": {
"fetch": AsyncMock(return_value=SAMPLE_AGGREGATED_RESPONSE),
"summarise": _summarise_usage_data,
"label": "global usage data",
}
},
)
def _usage_tool_handlers():
"""Tool handler override so no DB query is attempted, injected via `tool_handlers=`."""
return {
"get_usage_data": {
"fetch": AsyncMock(return_value=SAMPLE_AGGREGATED_RESPONSE),
"summarise": _summarise_usage_data,
"label": "global usage data",
}
}
class TestToolSchemas:
@ -482,25 +474,21 @@ class TestRouterResolution:
mock_router.acompletion = AsyncMock(
side_effect=[_make_tool_call_response(), _make_stream("Total spend is $50.25")]
)
mock_acompletion = AsyncMock()
with (
patch("litellm.proxy.proxy_server.llm_router", mock_router),
patch("litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.litellm") as mock_litellm,
_patched_usage_tool(),
):
mock_litellm.acompletion = AsyncMock()
events = await _collect_events(
messages=[{"role": "user", "content": "What is my total spend?"}],
model="my-gpt4",
user_id="user-123",
is_admin=True,
)
events = await _collect_events(
messages=[{"role": "user", "content": "What is my total spend?"}],
model="my-gpt4",
user_id="user-123",
is_admin=True,
deps=CompletionDeps(router=mock_router, acompletion=mock_acompletion),
tool_handlers=_usage_tool_handlers(),
)
assert [e for e in events if e["type"] == "error"] == []
assert [e["status"] for e in events if e["type"] == "tool_call"] == ["running", "complete"]
assert [e["content"] for e in events if e["type"] == "chunk"] == ["Total spend is $50.25"]
mock_litellm.acompletion.assert_not_called()
mock_acompletion.assert_not_called()
mock_router.get_model_list.assert_called_with(model_name="my-gpt4")
assert mock_router.acompletion.await_count == 2
@ -517,48 +505,90 @@ class TestRouterResolution:
mock_router = MagicMock()
mock_router.get_model_list.return_value = []
mock_router.acompletion = AsyncMock()
mock_acompletion = AsyncMock(side_effect=[_make_tool_call_response(), _make_stream("Spend summary")])
with (
patch("litellm.proxy.proxy_server.llm_router", mock_router),
patch("litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.litellm") as mock_litellm,
_patched_usage_tool(),
):
mock_litellm.acompletion = AsyncMock(
side_effect=[_make_tool_call_response(), _make_stream("Spend summary")]
)
events = await _collect_events(
messages=[{"role": "user", "content": "What is my total spend?"}],
model="openai/gpt-4o-mini",
user_id="user-123",
is_admin=True,
)
events = await _collect_events(
messages=[{"role": "user", "content": "What is my total spend?"}],
model="openai/gpt-4o-mini",
user_id="user-123",
is_admin=True,
deps=CompletionDeps(router=mock_router, acompletion=mock_acompletion),
tool_handlers=_usage_tool_handlers(),
)
assert [e for e in events if e["type"] == "error"] == []
mock_router.acompletion.assert_not_called()
assert mock_litellm.acompletion.await_count == 2
assert mock_litellm.acompletion.await_args_list[0].kwargs["model"] == "openai/gpt-4o-mini"
assert mock_acompletion.await_count == 2
assert mock_acompletion.await_args_list[0].kwargs["model"] == "openai/gpt-4o-mini"
@pytest.mark.asyncio
async def test_no_router_configured_uses_litellm(self):
with (
patch("litellm.proxy.proxy_server.llm_router", None),
patch("litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.litellm") as mock_litellm,
_patched_usage_tool(),
):
mock_litellm.acompletion = AsyncMock(
side_effect=[_make_tool_call_response(), _make_stream("Spend summary")]
)
mock_acompletion = AsyncMock(side_effect=[_make_tool_call_response(), _make_stream("Spend summary")])
events = await _collect_events(
messages=[{"role": "user", "content": "What is my total spend?"}],
model="gpt-4o-mini",
user_id="user-123",
is_admin=True,
)
events = await _collect_events(
messages=[{"role": "user", "content": "What is my total spend?"}],
model="gpt-4o-mini",
user_id="user-123",
is_admin=True,
deps=CompletionDeps(router=None, acompletion=mock_acompletion),
tool_handlers=_usage_tool_handlers(),
)
assert [e for e in events if e["type"] == "error"] == []
assert mock_litellm.acompletion.await_count == 2
assert mock_acompletion.await_count == 2
class TestUsageAiChatSpendAttribution:
"""
Regression: completions issued by the Usage dashboard's "Ask AI" chat must
carry the caller's key/user/team identity in `metadata`, the same as every
other proxied completion, so spend gets attributed and counts against the
caller's budget. Previously these completions carried no metadata at all,
so their cost was untracked regardless of which model handled them.
"""
@pytest.mark.asyncio
async def test_both_completions_carry_caller_identity_in_metadata(self):
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
mock_acompletion = AsyncMock(side_effect=[_make_tool_call_response(), _make_stream("Total spend is $50.25")])
await _collect_events(
messages=[{"role": "user", "content": "What is my total spend?"}],
model="gpt-4o-mini",
user_id="user-123",
is_admin=True,
deps=CompletionDeps(router=None, acompletion=mock_acompletion),
tool_handlers=_usage_tool_handlers(),
user_api_key_dict=UserAPIKeyAuth(
api_key="hashed-key",
user_id="user-123",
team_id="team-456",
user_role=LitellmUserRoles.INTERNAL_USER,
),
)
assert mock_acompletion.await_count == 2
for call in mock_acompletion.await_args_list:
assert call.kwargs["metadata"]["user_api_key_user_id"] == "user-123"
assert call.kwargs["metadata"]["user_api_key_team_id"] == "team-456"
assert call.kwargs["metadata"]["user_api_key"] == "hashed-key"
@pytest.mark.asyncio
async def test_metadata_is_empty_without_a_caller_identity(self):
mock_acompletion = AsyncMock(side_effect=[_make_tool_call_response(), _make_stream("Spend summary")])
await _collect_events(
messages=[{"role": "user", "content": "What is my total spend?"}],
model="gpt-4o-mini",
user_id="user-123",
is_admin=True,
deps=CompletionDeps(router=None, acompletion=mock_acompletion),
tool_handlers=_usage_tool_handlers(),
)
for call in mock_acompletion.await_args_list:
assert call.kwargs["metadata"] == {}
class TestUsageAiChatServiceAccountGuard:
@ -622,8 +652,9 @@ class TestUsageAiChatKeepalive:
async def _collect_endpoint_body(self, monkeypatch, interval, delay=0.3) -> tuple[list[bytes], dict]:
import asyncio
import litellm
from fastapi.responses import StreamingResponse
import litellm
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
from litellm.proxy.management_endpoints.usage_endpoints.endpoints import (
ChatMessage,

View file

@ -16781,7 +16781,6 @@ export interface paths {
* - permissions: Optional[dict] - [Not Implemented Yet] User-specific permissions, eg. turning off pii masking.
* - metadata: Optional[dict] - Metadata for user, store information for user. Example metadata = {"team": "core-infra", "app": "app2", "email": "ishaan@berri.ai" }
* - max_parallel_requests: Optional[int] - Rate limit a user based on the number of parallel requests. Raises 429 error, if user's parallel requests > x.
* - soft_budget: Optional[float] - Get alerts when user crosses given budget, doesn't block requests.
* - model_max_budget: Optional[dict] - Model-specific max budget for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-budgets-to-keys)
* - budget_fallbacks: Optional[Dict[str, List[str]]] - Per-model fallback chain tried in order when that model's own `model_max_budget` is exceeded, e.g. {"gpt-4o": ["gpt-4o-mini"]}.
* - model_rpm_limit: Optional[float] - Model-specific rpm limit for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-limits-to-keys)
@ -16887,7 +16886,6 @@ export interface paths {
* - permissions: Optional[dict] - [Not Implemented Yet] User-specific permissions, eg. turning off pii masking.
* - metadata: Optional[dict] - Metadata for user, store information for user. Example metadata = {"team": "core-infra", "app": "app2", "email": "ishaan@berri.ai" }
* - max_parallel_requests: Optional[int] - Rate limit a user based on the number of parallel requests. Raises 429 error, if user's parallel requests > x.
* - soft_budget: Optional[float] - Get alerts when user crosses given budget, doesn't block requests.
* - model_max_budget: Optional[dict] - Model-specific max budget for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-budgets-to-keys)
* - budget_fallbacks: Optional[Dict[str, List[str]]] - Per-model fallback chain tried in order when that model's own `model_max_budget` is exceeded, e.g. {"gpt-4o": ["gpt-4o-mini"]}.
* - model_rpm_limit: Optional[float] - Model-specific rpm limit for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-limits-to-keys)