mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
refactor: remove soft_budget field from user budget interface documentation
This commit is contained in:
parent
58b92fa7f8
commit
d3d5b9f37d
4 changed files with 157 additions and 79 deletions
|
|
@ -13,6 +13,9 @@ from typing_extensions import ReadOnly, TypedDict
|
|||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.constants import DEFAULT_COMPETITOR_DISCOVERY_MODEL
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup
|
||||
from litellm.router import Router
|
||||
from litellm.types.proxy.management_endpoints.common_daily_activity import (
|
||||
SpendAnalyticsPaginatedResponse,
|
||||
)
|
||||
|
|
@ -495,13 +498,14 @@ async def _process_tool_call(
|
|||
chat_messages: list[Mapping[str, object]],
|
||||
user_id: str | None,
|
||||
is_admin: bool,
|
||||
tool_handlers: Mapping[str, ToolHandler],
|
||||
) -> AsyncIterator[str]:
|
||||
"""Execute a single tool call, yielding SSE events for status."""
|
||||
fn_name: Final[str] = tc.function.name
|
||||
fn_args: Final[Mapping[str, str]] = json.loads(tc.function.arguments)
|
||||
|
||||
allowed_names: Final = {t["function"]["name"] for t in get_tools_for_role(is_admin)}
|
||||
handler: Final = TOOL_HANDLERS.get(fn_name)
|
||||
handler: Final = tool_handlers.get(fn_name)
|
||||
|
||||
if fn_name not in allowed_names or not handler:
|
||||
chat_messages.append(
|
||||
|
|
@ -532,7 +536,39 @@ async def _process_tool_call(
|
|||
chat_messages.append({"role": "tool", "tool_call_id": tc.id, "content": tool_result})
|
||||
|
||||
|
||||
def _resolve_completion_fn(model: str) -> Callable[..., Awaitable[ModelResponse | CustomStreamWrapper]]:
|
||||
class CompletionDeps(NamedTuple):
|
||||
"""Where to send a chat completion: the proxy's Router when it knows the model, else bare litellm."""
|
||||
|
||||
router: Router | None
|
||||
acompletion: Callable[..., Awaitable[ModelResponse | CustomStreamWrapper]]
|
||||
|
||||
|
||||
def _default_completion_deps() -> CompletionDeps:
|
||||
from litellm.proxy.proxy_server import llm_router
|
||||
|
||||
return CompletionDeps(router=llm_router, acompletion=litellm.acompletion)
|
||||
|
||||
|
||||
def _call_metadata(user_api_key_dict: UserAPIKeyAuth | None) -> Mapping[str, object]:
|
||||
"""Stamp the caller's key/user/team identity onto the completion call.
|
||||
|
||||
Without it, spend from these tool-planning and final-response completions is
|
||||
never attributed to the calling key/user/team, so it silently doesn't count
|
||||
against anyone's budget regardless of which model it is routed to.
|
||||
"""
|
||||
if user_api_key_dict is None:
|
||||
return {}
|
||||
stamped: Final = LiteLLMProxyRequestSetup.add_user_api_key_auth_to_request_metadata( # pyright: ignore[reportUnknownVariableType, reportUnknownMemberType] # untyped dict[Unknown, Unknown] signature upstream
|
||||
data={"metadata": {}},
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
_metadata_variable_name="metadata",
|
||||
)
|
||||
return cast(Mapping[str, object], stamped["metadata"])
|
||||
|
||||
|
||||
def _resolve_completion_fn(
|
||||
model: str, deps: CompletionDeps
|
||||
) -> Callable[..., Awaitable[ModelResponse | CustomStreamWrapper]]:
|
||||
"""Pick the Router when it knows the model, else a direct litellm call.
|
||||
|
||||
The "Ask AI" dropdown is populated from the proxy's model_list, so `model` is
|
||||
|
|
@ -543,23 +579,27 @@ def _resolve_completion_fn(model: str) -> Callable[..., Awaitable[ModelResponse
|
|||
with no Router and for strings the Router does not recognise, such as a genuine
|
||||
provider-prefixed model or an unregistered DEFAULT_COMPETITOR_DISCOVERY_MODEL.
|
||||
"""
|
||||
from litellm.proxy.proxy_server import llm_router
|
||||
|
||||
if llm_router is not None and llm_router.get_model_list(model_name=model):
|
||||
return llm_router.acompletion # pyright: ignore[reportUnknownVariableType] # untyped upstream signature
|
||||
return litellm.acompletion # pyright: ignore[reportUnknownVariableType] # untyped upstream signature
|
||||
if deps.router is not None and deps.router.get_model_list(model_name=model):
|
||||
return deps.router.acompletion # pyright: ignore[reportUnknownVariableType] # untyped upstream signature
|
||||
return deps.acompletion
|
||||
|
||||
|
||||
async def _stream_final_response(model: str, chat_messages: list[dict[str, Any]]) -> AsyncIterator[str]:
|
||||
async def _stream_final_response(
|
||||
model: str,
|
||||
chat_messages: list[dict[str, Any]],
|
||||
deps: CompletionDeps,
|
||||
metadata: Mapping[str, object],
|
||||
) -> AsyncIterator[str]:
|
||||
"""Stream the final LLM response after tool results are appended."""
|
||||
yield _sse({"type": "status", "message": "Analyzing results..."})
|
||||
|
||||
acompletion = _resolve_completion_fn(model)
|
||||
acompletion = _resolve_completion_fn(model, deps)
|
||||
response = await acompletion(
|
||||
model=model,
|
||||
messages=chat_messages,
|
||||
stream=True,
|
||||
temperature=USAGE_AI_TEMPERATURE,
|
||||
metadata=metadata,
|
||||
)
|
||||
async for chunk in response:
|
||||
delta = chunk.choices[0].delta.content
|
||||
|
|
@ -572,9 +612,16 @@ async def stream_usage_ai_chat(
|
|||
model: str | None = None,
|
||||
user_id: str | None = None,
|
||||
is_admin: bool = False,
|
||||
*,
|
||||
deps: CompletionDeps | None = None,
|
||||
tool_handlers: Mapping[str, ToolHandler] | None = None,
|
||||
user_api_key_dict: UserAPIKeyAuth | None = None,
|
||||
) -> AsyncGenerator[str, None]:
|
||||
"""Stream SSE events: status → tool_call → chunk → done."""
|
||||
resolved_deps: Final = deps if deps is not None else _default_completion_deps()
|
||||
resolved_handlers: Final = tool_handlers if tool_handlers is not None else TOOL_HANDLERS
|
||||
resolved_model: Final = (model or "").strip() or DEFAULT_COMPETITOR_DISCOVERY_MODEL
|
||||
metadata: Final = _call_metadata(user_api_key_dict)
|
||||
truncated: Final = messages[-MAX_CHAT_MESSAGES:] if len(messages) > MAX_CHAT_MESSAGES else messages
|
||||
chat_messages: Final[list[Mapping[str, object]]] = [
|
||||
{"role": "system", "content": _build_system_prompt(is_admin)},
|
||||
|
|
@ -584,12 +631,13 @@ async def stream_usage_ai_chat(
|
|||
try:
|
||||
yield _sse({"type": "status", "message": "Thinking..."})
|
||||
tools = get_tools_for_role(is_admin)
|
||||
acompletion = _resolve_completion_fn(resolved_model)
|
||||
acompletion = _resolve_completion_fn(resolved_model, resolved_deps)
|
||||
response = await acompletion(
|
||||
model=resolved_model,
|
||||
messages=chat_messages,
|
||||
tools=tools,
|
||||
temperature=USAGE_AI_TEMPERATURE,
|
||||
metadata=metadata,
|
||||
)
|
||||
choice: Final = response.choices[0]
|
||||
|
||||
|
|
@ -601,9 +649,9 @@ async def stream_usage_ai_chat(
|
|||
|
||||
chat_messages.append(choice.message.model_dump())
|
||||
for tc in choice.message.tool_calls:
|
||||
async for event in _process_tool_call(tc, chat_messages, user_id, is_admin):
|
||||
async for event in _process_tool_call(tc, chat_messages, user_id, is_admin, resolved_handlers):
|
||||
yield event
|
||||
async for event in _stream_final_response(resolved_model, chat_messages):
|
||||
async for event in _stream_final_response(resolved_model, chat_messages, resolved_deps, metadata):
|
||||
yield event
|
||||
yield _sse({"type": "done"})
|
||||
|
||||
|
|
|
|||
|
|
@ -67,6 +67,7 @@ async def usage_ai_chat(
|
|||
model=data.model,
|
||||
user_id=user_id,
|
||||
is_admin=is_admin,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
),
|
||||
ping_interval_seconds=litellm.sse_keepalive_ping_interval_seconds,
|
||||
ping_chunk=SSE_COMMENT_PING,
|
||||
|
|
|
|||
|
|
@ -8,16 +8,15 @@ from unittest.mock import AsyncMock, MagicMock, patch
|
|||
import pytest
|
||||
|
||||
from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import (
|
||||
TOOL_HANDLERS,
|
||||
TOOLS_ADMIN,
|
||||
TOOLS_BASE,
|
||||
CompletionDeps,
|
||||
_build_system_prompt,
|
||||
_summarise_entity_data,
|
||||
_summarise_usage_data,
|
||||
stream_usage_ai_chat,
|
||||
)
|
||||
|
||||
|
||||
SAMPLE_AGGREGATED_RESPONSE = {
|
||||
"results": [
|
||||
{
|
||||
|
|
@ -149,22 +148,15 @@ async def _collect_events(**kwargs):
|
|||
return [json.loads(event.removeprefix("data: ").strip()) async for event in stream_usage_ai_chat(**kwargs)]
|
||||
|
||||
|
||||
def _patched_usage_tool():
|
||||
"""Replace the get_usage_data handler so no DB query is attempted.
|
||||
|
||||
TOOL_HANDLERS captures `_fetch_usage_data` at import time, so patching the
|
||||
module attribute alone leaves the real (DB-backed) fetcher wired up.
|
||||
"""
|
||||
return patch.dict(
|
||||
"litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.TOOL_HANDLERS",
|
||||
{
|
||||
"get_usage_data": {
|
||||
"fetch": AsyncMock(return_value=SAMPLE_AGGREGATED_RESPONSE),
|
||||
"summarise": _summarise_usage_data,
|
||||
"label": "global usage data",
|
||||
}
|
||||
},
|
||||
)
|
||||
def _usage_tool_handlers():
|
||||
"""Tool handler override so no DB query is attempted, injected via `tool_handlers=`."""
|
||||
return {
|
||||
"get_usage_data": {
|
||||
"fetch": AsyncMock(return_value=SAMPLE_AGGREGATED_RESPONSE),
|
||||
"summarise": _summarise_usage_data,
|
||||
"label": "global usage data",
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class TestToolSchemas:
|
||||
|
|
@ -482,25 +474,21 @@ class TestRouterResolution:
|
|||
mock_router.acompletion = AsyncMock(
|
||||
side_effect=[_make_tool_call_response(), _make_stream("Total spend is $50.25")]
|
||||
)
|
||||
mock_acompletion = AsyncMock()
|
||||
|
||||
with (
|
||||
patch("litellm.proxy.proxy_server.llm_router", mock_router),
|
||||
patch("litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.litellm") as mock_litellm,
|
||||
_patched_usage_tool(),
|
||||
):
|
||||
mock_litellm.acompletion = AsyncMock()
|
||||
|
||||
events = await _collect_events(
|
||||
messages=[{"role": "user", "content": "What is my total spend?"}],
|
||||
model="my-gpt4",
|
||||
user_id="user-123",
|
||||
is_admin=True,
|
||||
)
|
||||
events = await _collect_events(
|
||||
messages=[{"role": "user", "content": "What is my total spend?"}],
|
||||
model="my-gpt4",
|
||||
user_id="user-123",
|
||||
is_admin=True,
|
||||
deps=CompletionDeps(router=mock_router, acompletion=mock_acompletion),
|
||||
tool_handlers=_usage_tool_handlers(),
|
||||
)
|
||||
|
||||
assert [e for e in events if e["type"] == "error"] == []
|
||||
assert [e["status"] for e in events if e["type"] == "tool_call"] == ["running", "complete"]
|
||||
assert [e["content"] for e in events if e["type"] == "chunk"] == ["Total spend is $50.25"]
|
||||
mock_litellm.acompletion.assert_not_called()
|
||||
mock_acompletion.assert_not_called()
|
||||
|
||||
mock_router.get_model_list.assert_called_with(model_name="my-gpt4")
|
||||
assert mock_router.acompletion.await_count == 2
|
||||
|
|
@ -517,48 +505,90 @@ class TestRouterResolution:
|
|||
mock_router = MagicMock()
|
||||
mock_router.get_model_list.return_value = []
|
||||
mock_router.acompletion = AsyncMock()
|
||||
mock_acompletion = AsyncMock(side_effect=[_make_tool_call_response(), _make_stream("Spend summary")])
|
||||
|
||||
with (
|
||||
patch("litellm.proxy.proxy_server.llm_router", mock_router),
|
||||
patch("litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.litellm") as mock_litellm,
|
||||
_patched_usage_tool(),
|
||||
):
|
||||
mock_litellm.acompletion = AsyncMock(
|
||||
side_effect=[_make_tool_call_response(), _make_stream("Spend summary")]
|
||||
)
|
||||
|
||||
events = await _collect_events(
|
||||
messages=[{"role": "user", "content": "What is my total spend?"}],
|
||||
model="openai/gpt-4o-mini",
|
||||
user_id="user-123",
|
||||
is_admin=True,
|
||||
)
|
||||
events = await _collect_events(
|
||||
messages=[{"role": "user", "content": "What is my total spend?"}],
|
||||
model="openai/gpt-4o-mini",
|
||||
user_id="user-123",
|
||||
is_admin=True,
|
||||
deps=CompletionDeps(router=mock_router, acompletion=mock_acompletion),
|
||||
tool_handlers=_usage_tool_handlers(),
|
||||
)
|
||||
|
||||
assert [e for e in events if e["type"] == "error"] == []
|
||||
mock_router.acompletion.assert_not_called()
|
||||
assert mock_litellm.acompletion.await_count == 2
|
||||
assert mock_litellm.acompletion.await_args_list[0].kwargs["model"] == "openai/gpt-4o-mini"
|
||||
assert mock_acompletion.await_count == 2
|
||||
assert mock_acompletion.await_args_list[0].kwargs["model"] == "openai/gpt-4o-mini"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_router_configured_uses_litellm(self):
|
||||
with (
|
||||
patch("litellm.proxy.proxy_server.llm_router", None),
|
||||
patch("litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.litellm") as mock_litellm,
|
||||
_patched_usage_tool(),
|
||||
):
|
||||
mock_litellm.acompletion = AsyncMock(
|
||||
side_effect=[_make_tool_call_response(), _make_stream("Spend summary")]
|
||||
)
|
||||
mock_acompletion = AsyncMock(side_effect=[_make_tool_call_response(), _make_stream("Spend summary")])
|
||||
|
||||
events = await _collect_events(
|
||||
messages=[{"role": "user", "content": "What is my total spend?"}],
|
||||
model="gpt-4o-mini",
|
||||
user_id="user-123",
|
||||
is_admin=True,
|
||||
)
|
||||
events = await _collect_events(
|
||||
messages=[{"role": "user", "content": "What is my total spend?"}],
|
||||
model="gpt-4o-mini",
|
||||
user_id="user-123",
|
||||
is_admin=True,
|
||||
deps=CompletionDeps(router=None, acompletion=mock_acompletion),
|
||||
tool_handlers=_usage_tool_handlers(),
|
||||
)
|
||||
|
||||
assert [e for e in events if e["type"] == "error"] == []
|
||||
assert mock_litellm.acompletion.await_count == 2
|
||||
assert mock_acompletion.await_count == 2
|
||||
|
||||
|
||||
class TestUsageAiChatSpendAttribution:
|
||||
"""
|
||||
Regression: completions issued by the Usage dashboard's "Ask AI" chat must
|
||||
carry the caller's key/user/team identity in `metadata`, the same as every
|
||||
other proxied completion, so spend gets attributed and counts against the
|
||||
caller's budget. Previously these completions carried no metadata at all,
|
||||
so their cost was untracked regardless of which model handled them.
|
||||
"""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_both_completions_carry_caller_identity_in_metadata(self):
|
||||
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
|
||||
|
||||
mock_acompletion = AsyncMock(side_effect=[_make_tool_call_response(), _make_stream("Total spend is $50.25")])
|
||||
|
||||
await _collect_events(
|
||||
messages=[{"role": "user", "content": "What is my total spend?"}],
|
||||
model="gpt-4o-mini",
|
||||
user_id="user-123",
|
||||
is_admin=True,
|
||||
deps=CompletionDeps(router=None, acompletion=mock_acompletion),
|
||||
tool_handlers=_usage_tool_handlers(),
|
||||
user_api_key_dict=UserAPIKeyAuth(
|
||||
api_key="hashed-key",
|
||||
user_id="user-123",
|
||||
team_id="team-456",
|
||||
user_role=LitellmUserRoles.INTERNAL_USER,
|
||||
),
|
||||
)
|
||||
|
||||
assert mock_acompletion.await_count == 2
|
||||
for call in mock_acompletion.await_args_list:
|
||||
assert call.kwargs["metadata"]["user_api_key_user_id"] == "user-123"
|
||||
assert call.kwargs["metadata"]["user_api_key_team_id"] == "team-456"
|
||||
assert call.kwargs["metadata"]["user_api_key"] == "hashed-key"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_metadata_is_empty_without_a_caller_identity(self):
|
||||
mock_acompletion = AsyncMock(side_effect=[_make_tool_call_response(), _make_stream("Spend summary")])
|
||||
|
||||
await _collect_events(
|
||||
messages=[{"role": "user", "content": "What is my total spend?"}],
|
||||
model="gpt-4o-mini",
|
||||
user_id="user-123",
|
||||
is_admin=True,
|
||||
deps=CompletionDeps(router=None, acompletion=mock_acompletion),
|
||||
tool_handlers=_usage_tool_handlers(),
|
||||
)
|
||||
|
||||
for call in mock_acompletion.await_args_list:
|
||||
assert call.kwargs["metadata"] == {}
|
||||
|
||||
|
||||
class TestUsageAiChatServiceAccountGuard:
|
||||
|
|
@ -622,8 +652,9 @@ class TestUsageAiChatKeepalive:
|
|||
async def _collect_endpoint_body(self, monkeypatch, interval, delay=0.3) -> tuple[list[bytes], dict]:
|
||||
import asyncio
|
||||
|
||||
import litellm
|
||||
from fastapi.responses import StreamingResponse
|
||||
|
||||
import litellm
|
||||
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
|
||||
from litellm.proxy.management_endpoints.usage_endpoints.endpoints import (
|
||||
ChatMessage,
|
||||
|
|
|
|||
2
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
2
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -16781,7 +16781,6 @@ export interface paths {
|
|||
* - permissions: Optional[dict] - [Not Implemented Yet] User-specific permissions, eg. turning off pii masking.
|
||||
* - metadata: Optional[dict] - Metadata for user, store information for user. Example metadata = {"team": "core-infra", "app": "app2", "email": "ishaan@berri.ai" }
|
||||
* - max_parallel_requests: Optional[int] - Rate limit a user based on the number of parallel requests. Raises 429 error, if user's parallel requests > x.
|
||||
* - soft_budget: Optional[float] - Get alerts when user crosses given budget, doesn't block requests.
|
||||
* - model_max_budget: Optional[dict] - Model-specific max budget for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-budgets-to-keys)
|
||||
* - budget_fallbacks: Optional[Dict[str, List[str]]] - Per-model fallback chain tried in order when that model's own `model_max_budget` is exceeded, e.g. {"gpt-4o": ["gpt-4o-mini"]}.
|
||||
* - model_rpm_limit: Optional[float] - Model-specific rpm limit for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-limits-to-keys)
|
||||
|
|
@ -16887,7 +16886,6 @@ export interface paths {
|
|||
* - permissions: Optional[dict] - [Not Implemented Yet] User-specific permissions, eg. turning off pii masking.
|
||||
* - metadata: Optional[dict] - Metadata for user, store information for user. Example metadata = {"team": "core-infra", "app": "app2", "email": "ishaan@berri.ai" }
|
||||
* - max_parallel_requests: Optional[int] - Rate limit a user based on the number of parallel requests. Raises 429 error, if user's parallel requests > x.
|
||||
* - soft_budget: Optional[float] - Get alerts when user crosses given budget, doesn't block requests.
|
||||
* - model_max_budget: Optional[dict] - Model-specific max budget for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-budgets-to-keys)
|
||||
* - budget_fallbacks: Optional[Dict[str, List[str]]] - Per-model fallback chain tried in order when that model's own `model_max_budget` is exceeded, e.g. {"gpt-4o": ["gpt-4o-mini"]}.
|
||||
* - model_rpm_limit: Optional[float] - Model-specific rpm limit for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-limits-to-keys)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue