mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
Merge 41334bfcfe into 837f36e872
This commit is contained in:
commit
ec86fb239e
5 changed files with 194 additions and 11 deletions
|
|
@ -7145,6 +7145,8 @@ async def _check_team_member_budget(
|
|||
)
|
||||
|
||||
if team_member_budget is not None:
|
||||
valid_token.max_budget_in_team = team_member_budget
|
||||
valid_token.team_member_max_budget = team_member_budget
|
||||
team_member_spend = (loaded_membership.spend if loaded_membership is not None else 0.0) or 0.0
|
||||
|
||||
# Read from cross-pod counter (Redis-first) if available
|
||||
|
|
|
|||
|
|
@ -11,9 +11,9 @@ attempted, mirroring `fallback_model_access.py`. It deliberately leaves the prim
|
|||
a zero-cost model is never blocked by budget, and only the paid fallback is refused. On by default;
|
||||
set `general_settings.enforce_fallback_budget: false` to restore the unguarded behaviour.
|
||||
|
||||
Scope: the key's and the user's `max_budget`, plus the team member's per-model caps. Not covered yet,
|
||||
and each needs a read-only evaluation path before it can be: team and team-member overall, end-user,
|
||||
org, global and other per-model budgets, whose auth-path functions enforce rather than report (they
|
||||
Scope: the key's and the user's `max_budget`, the team member's overall budget (`max_budget_in_team`),
|
||||
plus the team member's per-model caps. Not covered yet, and each needs a read-only evaluation path before
|
||||
it can be: team overall, end-user, org, global and other per-model budgets, whose auth-path functions enforce rather than report (they
|
||||
raise), so reusing them would fire threshold alerts and take spend reservations for a target that is
|
||||
then skipped; and the key's rolling `budget_limits` windows, whose accumulated spend lives only in
|
||||
per-window counters (`spend:key:{token}:window:{budget_duration}`), so enforcing them means more
|
||||
|
|
@ -156,6 +156,19 @@ async def is_token_within_budget_for_model(
|
|||
if user_spend >= user_budget:
|
||||
return False
|
||||
|
||||
member_team_budget: Final = getattr(valid_token, "max_budget_in_team", None)
|
||||
team_member_budget: Final = (
|
||||
member_team_budget if member_team_budget is not None else getattr(valid_token, "team_member_max_budget", None)
|
||||
)
|
||||
if team_member_budget is not None and valid_token.user_id is not None and valid_token.team_id is not None:
|
||||
team_member_spend: Final = await _counter_spend(
|
||||
counter_key=f"spend:team_member:{valid_token.user_id}:{valid_token.team_id}",
|
||||
fallback_spend=valid_token.team_member_spend or 0.0,
|
||||
max_budget=team_member_budget,
|
||||
)
|
||||
if team_member_spend >= team_member_budget:
|
||||
return False
|
||||
|
||||
team_member_model_max_budget: Final = valid_token.team_member_model_max_budget
|
||||
if (
|
||||
team_member_model_budget_limiter is not None
|
||||
|
|
|
|||
|
|
@ -3074,7 +3074,9 @@ async def validate_resolved_virtual_key( # noqa: C901 # Preserve ordering of e
|
|||
team_member_budget: Final = team_member_info.litellm_budget_table.effective_max_budget(
|
||||
now=datetime.now(timezone.utc),
|
||||
)
|
||||
if team_member_budget is not None and team_member_budget > 0:
|
||||
if team_member_budget is not None:
|
||||
valid_token.max_budget_in_team = team_member_budget
|
||||
valid_token.team_member_max_budget = team_member_budget
|
||||
# Read from cross-pod counter (Redis-first) if available
|
||||
from litellm.proxy.proxy_server import get_current_spend
|
||||
|
||||
|
|
@ -3504,6 +3506,15 @@ async def _load_team_member_default_model_budget(
|
|||
user_api_key_cache=user_api_key_cache,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
)
|
||||
if membership is not None:
|
||||
user_api_key_auth_obj.team_member_spend = membership.spend
|
||||
member_budget_row_early: Final = membership.litellm_budget_table
|
||||
if member_budget_row_early is not None:
|
||||
member_budget: Final = member_budget_row_early.effective_max_budget(now=datetime.now(timezone.utc))
|
||||
if member_budget is not None:
|
||||
user_api_key_auth_obj.max_budget_in_team = member_budget
|
||||
user_api_key_auth_obj.team_member_max_budget = member_budget
|
||||
|
||||
member_budget_row: Final = membership.litellm_budget_table if membership is not None else None
|
||||
member_budget_value: Final[object] = (
|
||||
getattr(member_budget_row, "model_max_budget", None) if member_budget_row is not None else None
|
||||
|
|
@ -3527,6 +3538,20 @@ async def _load_team_member_default_model_budget(
|
|||
user_api_key_cache=user_api_key_cache,
|
||||
raise_on_lookup_error=True,
|
||||
)
|
||||
# Team-level default budget of 0 denotes unset / no cap (individual membership row can explicitly set 0.0 to block).
|
||||
if (
|
||||
user_api_key_auth_obj.max_budget_in_team is None
|
||||
and team_member_default_budget is not None
|
||||
and team_member_default_budget.max_budget is not None
|
||||
and team_member_default_budget.max_budget > 0
|
||||
):
|
||||
default_max: Final = team_member_default_budget.max_budget + (
|
||||
member_budget_row.active_temp_budget_increase(now=datetime.now(timezone.utc))
|
||||
if member_budget_row is not None
|
||||
else 0.0
|
||||
)
|
||||
user_api_key_auth_obj.max_budget_in_team = default_max
|
||||
user_api_key_auth_obj.team_member_max_budget = default_max
|
||||
except Exception:
|
||||
if not skip_budget_checks:
|
||||
raise
|
||||
|
|
|
|||
|
|
@ -78,6 +78,8 @@ class LiteLLM_VerificationTokenView(LiteLLM_VerificationToken):
|
|||
team_member_spend: float | None = None
|
||||
team_member_tpm_limit: int | None = None
|
||||
team_member_rpm_limit: int | None = None
|
||||
max_budget_in_team: float | None = None
|
||||
team_member_max_budget: float | None = None
|
||||
|
||||
# End User Params
|
||||
end_user_id: str | None = None
|
||||
|
|
|
|||
|
|
@ -1,8 +1,7 @@
|
|||
from collections.abc import Mapping, Sequence
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import litellm
|
||||
import pytest
|
||||
|
||||
from litellm import Router
|
||||
|
|
@ -116,6 +115,151 @@ async def test_paid_target_refused_when_over_user_budget():
|
|||
assert await is_token_within_budget_for_model(model="paid-model", valid_token=token, llm_router=_router()) is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_paid_target_refused_when_over_team_member_overall_budget():
|
||||
token = _token(user_id="u1", team_id="t1", max_budget_in_team=50.0, team_member_spend=100.0)
|
||||
assert await is_token_within_budget_for_model(model="paid-model", valid_token=token, llm_router=_router()) is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_paid_target_allowed_when_under_team_member_overall_budget():
|
||||
token = _token(user_id="u1", team_id="t1", max_budget_in_team=50.0, team_member_spend=10.0)
|
||||
assert await is_token_within_budget_for_model(model="paid-model", valid_token=token, llm_router=_router()) is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_free_target_allowed_even_when_over_team_member_overall_budget():
|
||||
token = _token(user_id="u1", team_id="t1", max_budget_in_team=50.0, team_member_spend=100.0)
|
||||
assert await is_token_within_budget_for_model(model="free-model", valid_token=token, llm_router=_router()) is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_paid_target_refused_when_team_member_budget_is_zero():
|
||||
token = _token(user_id="u1", team_id="t1", max_budget_in_team=0.0, team_member_spend=0.0)
|
||||
assert await is_token_within_budget_for_model(model="paid-model", valid_token=token, llm_router=_router()) is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_load_team_member_budget_populates_token_when_budget_checks_skipped():
|
||||
from litellm.proxy.auth.user_api_key_auth import _load_team_member_default_model_budget
|
||||
|
||||
token = _token(user_id="u1", team_id="t1")
|
||||
assert token.max_budget_in_team is None
|
||||
|
||||
mock_membership = MagicMock()
|
||||
mock_membership.spend = 15.0
|
||||
mock_budget_row = MagicMock()
|
||||
mock_budget_row.effective_max_budget.return_value = 50.0
|
||||
mock_budget_row.model_max_budget = None
|
||||
mock_membership.litellm_budget_table = mock_budget_row
|
||||
mock_membership.budget_id = None
|
||||
|
||||
with patch("litellm.proxy.auth.user_api_key_auth.get_team_membership", new=AsyncMock(return_value=mock_membership)):
|
||||
await _load_team_member_default_model_budget(
|
||||
user_api_key_auth_obj=token,
|
||||
team_object=None,
|
||||
prisma_client=MagicMock(),
|
||||
user_api_key_cache=MagicMock(),
|
||||
skip_budget_checks=True,
|
||||
proxy_logging_obj=None,
|
||||
)
|
||||
|
||||
assert token.max_budget_in_team == 50.0
|
||||
assert token.team_member_spend == 15.0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_load_team_member_default_budget_zero_treated_as_no_cap():
|
||||
from litellm.proxy.auth.user_api_key_auth import _load_team_member_default_model_budget
|
||||
|
||||
token = _token(user_id="u1", team_id="t1")
|
||||
assert token.max_budget_in_team is None
|
||||
|
||||
mock_team = MagicMock()
|
||||
mock_team.metadata = {"team_member_budget_id": "b-default"}
|
||||
|
||||
mock_default_budget = MagicMock()
|
||||
mock_default_budget.max_budget = 0.0
|
||||
|
||||
with (
|
||||
patch("litellm.proxy.auth.user_api_key_auth.get_team_membership", new=AsyncMock(return_value=None)),
|
||||
patch(
|
||||
"litellm.proxy.auth.user_api_key_auth.get_team_member_default_budget",
|
||||
new=AsyncMock(return_value=mock_default_budget),
|
||||
),
|
||||
):
|
||||
await _load_team_member_default_model_budget(
|
||||
user_api_key_auth_obj=token,
|
||||
team_object=mock_team,
|
||||
prisma_client=MagicMock(),
|
||||
user_api_key_cache=MagicMock(),
|
||||
skip_budget_checks=True,
|
||||
proxy_logging_obj=None,
|
||||
)
|
||||
|
||||
assert token.max_budget_in_team is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_e2e_auth_zero_cost_primary_populates_member_budget_and_refuses_paid_fallback(monkeypatch):
|
||||
"""
|
||||
End-to-end auth -> fallback test (#46874):
|
||||
Run the auth path (run_centralized_common_checks) with a zero-cost primary model
|
||||
(so skip_budget_checks=True), verify team member budget and spend are populated onto the token,
|
||||
and assert is_token_within_budget_for_model refuses the paid fallback target when spend >= budget.
|
||||
"""
|
||||
from fastapi import Request
|
||||
from starlette.datastructures import URL
|
||||
|
||||
import litellm.proxy.proxy_server as _proxy_server_mod
|
||||
from litellm.proxy.auth.user_api_key_auth import run_centralized_common_checks
|
||||
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
|
||||
|
||||
router = _router()
|
||||
token = _token(user_id="u1", team_id="t1")
|
||||
assert token.max_budget_in_team is None
|
||||
assert token.team_member_spend is None
|
||||
|
||||
mock_membership = MagicMock()
|
||||
mock_membership.spend = 50.0
|
||||
mock_budget_row = MagicMock()
|
||||
mock_budget_row.effective_max_budget.return_value = 50.0
|
||||
mock_budget_row.model_max_budget = None
|
||||
mock_membership.litellm_budget_table = mock_budget_row
|
||||
mock_membership.budget_id = None
|
||||
|
||||
key_cache = UserApiKeyCache()
|
||||
monkeypatch.setattr(_proxy_server_mod, "llm_router", router, raising=False)
|
||||
monkeypatch.setattr(_proxy_server_mod, "general_settings", {}, raising=False)
|
||||
monkeypatch.setattr(_proxy_server_mod, "prisma_client", MagicMock(), raising=False)
|
||||
monkeypatch.setattr(_proxy_server_mod, "user_api_key_cache", key_cache, raising=False)
|
||||
monkeypatch.setattr(_proxy_server_mod, "proxy_logging_obj", None, raising=False)
|
||||
monkeypatch.setattr(_proxy_server_mod, "master_key", "sk-master", raising=False)
|
||||
|
||||
request = Request(scope={"type": "http"})
|
||||
request._url = URL(url="/chat/completions")
|
||||
|
||||
with (
|
||||
patch("litellm.proxy.auth.user_api_key_auth.get_team_membership", new=AsyncMock(return_value=mock_membership)),
|
||||
patch("litellm.proxy.auth.user_api_key_auth.get_team_object", new=AsyncMock(return_value=None)),
|
||||
patch("litellm.proxy.auth.user_api_key_auth.get_user_object", new=AsyncMock(return_value=None)),
|
||||
patch("litellm.proxy.auth.user_api_key_auth.common_checks", new=AsyncMock(return_value=True)),
|
||||
):
|
||||
await run_centralized_common_checks(
|
||||
user_api_key_auth_obj=token,
|
||||
request=request,
|
||||
request_data={"model": "free-model", "messages": [{"role": "user", "content": "hi"}]},
|
||||
route="/chat/completions",
|
||||
)
|
||||
|
||||
# 1. Token has team member budget and spend stamped despite zero-cost skip
|
||||
assert token.max_budget_in_team == 50.0
|
||||
assert token.team_member_spend == 50.0
|
||||
|
||||
# 2. Fallback evaluator refuses the paid fallback target because spend >= budget
|
||||
assert await is_token_within_budget_for_model(model="paid-model", valid_token=token, llm_router=router) is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_team_member_budget_refuses_an_exhausted_paid_target() -> None:
|
||||
member_caps: Final = {"paid-model": {"max_budget": 5.0, "budget_duration": "1d"}}
|
||||
|
|
@ -450,9 +594,7 @@ async def test_router_skips_the_paid_fallback_target_when_over_budget():
|
|||
is False
|
||||
)
|
||||
assert (
|
||||
await _is_fallback_target_within_budget(
|
||||
router, "paid-model", "free-model", under, attempted=under_attempted
|
||||
)
|
||||
await _is_fallback_target_within_budget(router, "paid-model", "free-model", under, attempted=under_attempted)
|
||||
is True
|
||||
)
|
||||
assert over_attempted.skipped == (SkippedFallbackTarget("paid-model", FallbackSkipReason.OVER_BUDGET),)
|
||||
|
|
@ -467,8 +609,7 @@ async def test_router_without_a_budget_check_attempts_every_fallback():
|
|||
attempted: Final = AttemptedFallbackTargets()
|
||||
|
||||
assert (
|
||||
await _is_fallback_target_within_budget(router, "paid-model", "free-model", over, attempted=attempted)
|
||||
is True
|
||||
await _is_fallback_target_within_budget(router, "paid-model", "free-model", over, attempted=attempted) is True
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue