mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(proxy): enforce per-model key budgets across key and team model aliases
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
50cd26cd9c
commit
22e2e39628
6 changed files with 206 additions and 22 deletions
|
|
@ -225,14 +225,30 @@ async def _normalize_claude_model(
|
|||
|
||||
def _get_model_names_for_budget_checks(
|
||||
model: str | list[str] | None,
|
||||
aliases: Mapping[str, str] | None = None,
|
||||
) -> list[str]:
|
||||
if model is None:
|
||||
return []
|
||||
if isinstance(model, str):
|
||||
return [model]
|
||||
alias: Final = aliases.get(model) if isinstance(aliases, Mapping) else None
|
||||
return [model, alias] if isinstance(alias, str) and alias != model else [model]
|
||||
return model
|
||||
|
||||
|
||||
def _get_model_names_for_token_budget_checks(
|
||||
model: str | list[str] | None,
|
||||
valid_token: UserAPIKeyAuth,
|
||||
) -> list[str]:
|
||||
return list(
|
||||
dict.fromkeys(
|
||||
(
|
||||
*_get_model_names_for_budget_checks(model=model, aliases=valid_token.aliases),
|
||||
*_get_model_names_for_budget_checks(model=model, aliases=valid_token.team_model_aliases),
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
class _KeyModelBudgetLimiter(Protocol):
|
||||
async def is_key_within_model_budget(self, user_api_key_dict: UserAPIKeyAuth, model: str) -> bool: ...
|
||||
|
||||
|
|
@ -308,6 +324,7 @@ async def _check_key_model_budget_with_fallback(
|
|||
valid_token: UserAPIKeyAuth,
|
||||
model_max_budget_limiter: _KeyModelBudgetLimiter,
|
||||
model_name: str,
|
||||
requested_model: str,
|
||||
request_data: dict,
|
||||
request: Request,
|
||||
llm_model_list: list | None = None,
|
||||
|
|
@ -327,7 +344,7 @@ async def _check_key_model_budget_with_fallback(
|
|||
regardless of whether they consume ``_read_request_body()``,
|
||||
``request.json()``, or the path ``model`` parameter.
|
||||
|
||||
Fallback is only attempted when ``model_name`` matches the top-level
|
||||
Fallback is only attempted when ``requested_model`` matches the top-level
|
||||
``request_data["model"]``; models extracted from nested fields
|
||||
(``session.model``, ``completion.model``, etc.) are not rewritable
|
||||
and raise immediately.
|
||||
|
|
@ -342,7 +359,7 @@ async def _check_key_model_budget_with_fallback(
|
|||
model=model_name,
|
||||
)
|
||||
except litellm.BudgetExceededError as e:
|
||||
if request_data.get("model") != model_name:
|
||||
if request_data.get("model") != requested_model:
|
||||
raise e
|
||||
fallback_model: Final = await model_max_budget_limiter.get_fallback_model_within_budget(
|
||||
user_api_key_dict=valid_token,
|
||||
|
|
@ -1618,13 +1635,14 @@ async def _user_api_key_auth_builder(
|
|||
await _check_user_model_budget(
|
||||
valid_token=cast(UserAPIKeyAuth, valid_token),
|
||||
model_max_budget_limiter=model_max_budget_limiter,
|
||||
models=_get_model_names_for_budget_checks(
|
||||
models=_get_model_names_for_token_budget_checks(
|
||||
model=_get_model_from_request_context(
|
||||
request_data=request_data,
|
||||
route=route,
|
||||
request=request,
|
||||
llm_router=llm_router,
|
||||
)
|
||||
),
|
||||
valid_token=valid_token,
|
||||
),
|
||||
)
|
||||
|
||||
|
|
@ -2130,7 +2148,10 @@ async def _user_api_key_auth_builder(
|
|||
request=request,
|
||||
llm_router=llm_router,
|
||||
)
|
||||
current_models = _get_model_names_for_budget_checks(model=current_model)
|
||||
current_models = _get_model_names_for_token_budget_checks(
|
||||
model=current_model,
|
||||
valid_token=valid_token,
|
||||
)
|
||||
|
||||
if (
|
||||
max_budget_per_model is not None
|
||||
|
|
@ -2146,6 +2167,7 @@ async def _user_api_key_auth_builder(
|
|||
valid_token=valid_token,
|
||||
model_max_budget_limiter=model_max_budget_limiter,
|
||||
model_name=model_name,
|
||||
requested_model=current_model if isinstance(current_model, str) else model_name,
|
||||
request_data=request_data,
|
||||
request=request,
|
||||
llm_model_list=llm_model_list,
|
||||
|
|
@ -2160,7 +2182,10 @@ async def _user_api_key_auth_builder(
|
|||
request=request,
|
||||
llm_router=llm_router,
|
||||
)
|
||||
current_models = _get_model_names_for_budget_checks(model=current_model)
|
||||
current_models = _get_model_names_for_token_budget_checks(
|
||||
model=current_model,
|
||||
valid_token=valid_token,
|
||||
)
|
||||
|
||||
# Check 5a. Internal user model_max_budget
|
||||
if current_models:
|
||||
|
|
@ -3298,7 +3323,7 @@ async def _run_post_custom_auth_checks(
|
|||
request=request,
|
||||
llm_router=llm_router,
|
||||
)
|
||||
current_models = _get_model_names_for_budget_checks(model=current_model)
|
||||
current_models = _get_model_names_for_token_budget_checks(model=current_model, valid_token=valid_token)
|
||||
|
||||
# A zero-cost model cannot move any counter, so refusing it means refusing on
|
||||
# spend some other model accrued. The JWT and virtual-key paths already skip
|
||||
|
|
@ -3325,6 +3350,7 @@ async def _run_post_custom_auth_checks(
|
|||
valid_token=valid_token,
|
||||
model_max_budget_limiter=model_max_budget_limiter,
|
||||
model_name=model_name,
|
||||
requested_model=current_model if isinstance(current_model, str) else model_name,
|
||||
request_data=request_data,
|
||||
request=request,
|
||||
llm_model_list=llm_model_list,
|
||||
|
|
@ -3339,7 +3365,7 @@ async def _run_post_custom_auth_checks(
|
|||
request=request,
|
||||
llm_router=llm_router,
|
||||
)
|
||||
current_models = _get_model_names_for_budget_checks(model=current_model)
|
||||
current_models = _get_model_names_for_token_budget_checks(model=current_model, valid_token=valid_token)
|
||||
|
||||
# 3b. Attach and check the internal user's model_max_budget.
|
||||
# Custom auth builds its own token, so unlike the main path nothing has
|
||||
|
|
|
|||
|
|
@ -245,19 +245,34 @@ def _as_spend(current_spend: object) -> float:
|
|||
|
||||
|
||||
def _resolve_entity_model_budgets(
|
||||
model: str,
|
||||
models: tuple[str, ...],
|
||||
entity_budgets: Iterable[tuple[Litellm_EntityType, str | None, object]],
|
||||
) -> tuple[tuple[Litellm_EntityType, str, ResolvedModelBudget], ...]:
|
||||
"""Drop the scopes that do not budget `model`, keeping only what can be incremented."""
|
||||
"""Drop scopes that do not budget any request model, keeping only what can be incremented."""
|
||||
return tuple(
|
||||
(entity_type, entity_id, resolved)
|
||||
for entity_type, entity_id, model_max_budget in entity_budgets
|
||||
if entity_id is not None and isinstance(model_max_budget, Mapping) and model_max_budget
|
||||
for resolved in (resolve_model_budget(model=model, model_max_budget=model_max_budget),)
|
||||
for resolved in (_first_resolved(models=models, model_max_budget=model_max_budget),)
|
||||
if resolved is not None and resolved.budget_config.budget_duration is not None
|
||||
)
|
||||
|
||||
|
||||
def _first_resolved(
|
||||
models: tuple[str, ...],
|
||||
model_max_budget: Mapping[str, object],
|
||||
) -> ResolvedModelBudget | None:
|
||||
return next(
|
||||
(
|
||||
resolved
|
||||
for model in models
|
||||
for resolved in (resolve_model_budget(model=model, model_max_budget=model_max_budget),)
|
||||
if resolved is not None and resolved.budget_config.budget_duration is not None
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
|
||||
"""
|
||||
Handles budgets for model + virtual key
|
||||
|
|
@ -298,7 +313,15 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
|
|||
model: str,
|
||||
) -> str | None:
|
||||
budget_fallbacks: Final[dict[str, list[str]]] = user_api_key_dict.budget_fallbacks or {}
|
||||
for fallback_model in budget_fallbacks.get(model, []):
|
||||
fallback_models: Final = next(
|
||||
(
|
||||
budget_fallbacks[candidate]
|
||||
for candidate in _budget_model_candidates(model)
|
||||
if candidate in budget_fallbacks
|
||||
),
|
||||
(),
|
||||
)
|
||||
for fallback_model in fallback_models:
|
||||
try:
|
||||
await self.is_key_within_model_budget(user_api_key_dict=user_api_key_dict, model=fallback_model)
|
||||
return fallback_model
|
||||
|
|
@ -445,16 +468,12 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
|
|||
_metadata: Final[dict] = _litellm_params.get("metadata", {}) or {}
|
||||
payload_metadata: Final = standard_logging_payload.get("metadata") or {}
|
||||
|
||||
# Use model_group (the user-facing model alias, e.g. "gpt-4o") when
|
||||
# available. The enforcement path receives the model name from
|
||||
# request_data["model"] which is the model group alias, so the spend
|
||||
# tracking cache key must resolve from the same name. Falling back to
|
||||
# the deployment-level "model" field preserves behaviour for non-proxy
|
||||
# or non-router deployments where model_group is None.
|
||||
model: Final = standard_logging_payload.get("model_group") or standard_logging_payload.get("model")
|
||||
if model is None:
|
||||
return
|
||||
|
||||
requested_model: Final = _metadata.get("litellm_client_requested_model")
|
||||
models: Final = tuple(dict.fromkeys(m for m in (model, requested_model) if isinstance(m, str)))
|
||||
response_cost: Final[float] = standard_logging_payload.get("response_cost", 0)
|
||||
entity_budgets: Final = (
|
||||
(
|
||||
|
|
@ -474,12 +493,12 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
|
|||
),
|
||||
)
|
||||
|
||||
resolved_budgets: Final = _resolve_entity_model_budgets(model=model, entity_budgets=entity_budgets)
|
||||
resolved_budgets: Final = _resolve_entity_model_budgets(models=models, entity_budgets=entity_budgets)
|
||||
if not resolved_budgets:
|
||||
verbose_proxy_logger.debug(
|
||||
"Not running _PROXY_VirtualKeyModelMaxBudgetLimiter.async_log_success_event: "
|
||||
"no key, user or end-user model_max_budget covers model=%s",
|
||||
model,
|
||||
"no key, user or end-user model_max_budget covers models=%s",
|
||||
models,
|
||||
)
|
||||
return
|
||||
|
||||
|
|
|
|||
|
|
@ -2439,6 +2439,9 @@ async def add_litellm_data_to_request(
|
|||
user_api_key_dict=user_api_key_dict,
|
||||
)
|
||||
|
||||
if isinstance(_pre_alias_model, str) and data.get("model") != _pre_alias_model:
|
||||
data[_metadata_variable_name]["litellm_client_requested_model"] = _pre_alias_model # rebind-ok: metadata out-param
|
||||
|
||||
verbose_proxy_logger.debug("[PROXY] returned data from litellm_pre_call_utils: %s", data)
|
||||
|
||||
# Team/Project credential overrides from model_config
|
||||
|
|
|
|||
|
|
@ -35,6 +35,7 @@ from litellm.proxy.auth.auth_checks import get_key_object, _cache_key_object
|
|||
from litellm.proxy.auth.route_checks import RouteChecks
|
||||
from litellm.proxy.auth.user_api_key_auth import (
|
||||
_check_key_model_budget_with_fallback,
|
||||
_get_model_names_for_budget_checks,
|
||||
_ensure_litellm_received_at_on_request_state,
|
||||
_ensure_parent_otel_span_on_request_state,
|
||||
_PendingAutoRegister,
|
||||
|
|
@ -50,6 +51,13 @@ from litellm.proxy.auth.user_api_key_auth import (
|
|||
)
|
||||
|
||||
|
||||
def test_get_model_names_for_budget_checks_includes_alias_target():
|
||||
assert _get_model_names_for_budget_checks(
|
||||
"claude-haiku-4-5",
|
||||
aliases={"claude-haiku-4-5": "claude-haiku-4-5-20251001"},
|
||||
) == ["claude-haiku-4-5", "claude-haiku-4-5-20251001"]
|
||||
|
||||
|
||||
class _RoutingRequest:
|
||||
def __init__(self, headers=None, query_params=None):
|
||||
self.headers = headers or {}
|
||||
|
|
@ -6399,6 +6407,7 @@ class TestCheckKeyModelBudgetWithFallback:
|
|||
valid_token=valid_token,
|
||||
model_max_budget_limiter=limiter,
|
||||
model_name="gpt-4o",
|
||||
requested_model="gpt-4o",
|
||||
request_data=request_data,
|
||||
request=request,
|
||||
)
|
||||
|
|
@ -6425,6 +6434,7 @@ class TestCheckKeyModelBudgetWithFallback:
|
|||
valid_token=valid_token,
|
||||
model_max_budget_limiter=limiter,
|
||||
model_name="gpt-4o",
|
||||
requested_model="gpt-4o",
|
||||
request_data=request_data,
|
||||
request=request,
|
||||
)
|
||||
|
|
@ -6439,6 +6449,31 @@ class TestCheckKeyModelBudgetWithFallback:
|
|||
cached_keys, cached_body = request.scope["parsed_body"]
|
||||
assert cached_body["model"] == "gpt-4o-mini"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_alias_target_budget_reroutes_original_request_to_fallback(self):
|
||||
valid_token = UserAPIKeyAuth(
|
||||
token="test-key",
|
||||
budget_fallbacks={"claude-haiku-4-5-20251001": ["claude-sonnet-4-5"]},
|
||||
)
|
||||
limiter = AsyncMock()
|
||||
limiter.is_key_within_model_budget.side_effect = litellm.BudgetExceededError(
|
||||
current_cost=10, max_budget=5
|
||||
)
|
||||
limiter.get_fallback_model_within_budget.return_value = "claude-sonnet-4-5"
|
||||
request_data = {"model": "claude-haiku-4-5"}
|
||||
request = self._make_request()
|
||||
|
||||
await _check_key_model_budget_with_fallback(
|
||||
valid_token=valid_token,
|
||||
model_max_budget_limiter=limiter,
|
||||
model_name="claude-haiku-4-5-20251001",
|
||||
requested_model="claude-haiku-4-5",
|
||||
request_data=request_data,
|
||||
request=request,
|
||||
)
|
||||
|
||||
assert request_data["model"] == "claude-sonnet-4-5"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_raises_when_every_fallback_also_exceeded(self):
|
||||
valid_token = UserAPIKeyAuth(
|
||||
|
|
@ -6456,6 +6491,7 @@ class TestCheckKeyModelBudgetWithFallback:
|
|||
valid_token=valid_token,
|
||||
model_max_budget_limiter=limiter,
|
||||
model_name="gpt-4o",
|
||||
requested_model="gpt-4o",
|
||||
request_data=request_data,
|
||||
request=request,
|
||||
)
|
||||
|
|
@ -6494,6 +6530,7 @@ class TestCheckKeyModelBudgetWithFallback:
|
|||
valid_token=valid_token,
|
||||
model_max_budget_limiter=limiter,
|
||||
model_name="gpt-4o",
|
||||
requested_model="gpt-4o",
|
||||
request_data=request_data,
|
||||
request=request,
|
||||
llm_model_list=None,
|
||||
|
|
@ -6528,6 +6565,7 @@ class TestCheckKeyModelBudgetWithFallback:
|
|||
valid_token=valid_token,
|
||||
model_max_budget_limiter=limiter,
|
||||
model_name="gpt-4o",
|
||||
requested_model="gpt-4o",
|
||||
request_data=request_data,
|
||||
request=request,
|
||||
llm_model_list=None,
|
||||
|
|
@ -6563,6 +6601,7 @@ class TestCheckKeyModelBudgetWithFallback:
|
|||
valid_token=valid_token,
|
||||
model_max_budget_limiter=limiter,
|
||||
model_name="gpt-4o",
|
||||
requested_model="gpt-4o",
|
||||
request_data=request_data,
|
||||
request=request,
|
||||
llm_model_list=None,
|
||||
|
|
@ -6599,6 +6638,7 @@ class TestCheckKeyModelBudgetWithFallback:
|
|||
valid_token=valid_token,
|
||||
model_max_budget_limiter=limiter,
|
||||
model_name="gpt-4o",
|
||||
requested_model="gpt-4o",
|
||||
request_data=request_data,
|
||||
request=request,
|
||||
llm_model_list=None,
|
||||
|
|
@ -6629,6 +6669,7 @@ class TestCheckKeyModelBudgetWithFallback:
|
|||
valid_token=valid_token,
|
||||
model_max_budget_limiter=limiter,
|
||||
model_name="gpt-4o",
|
||||
requested_model="gpt-4o",
|
||||
request_data=request_data,
|
||||
request=request,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ from fastapi import HTTPException
|
|||
from litellm.caching.caching import DualCache
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.hooks.max_budget_limiter import _PROXY_MaxBudgetLimiter
|
||||
from litellm.proxy.hooks.model_max_budget_limiter import _PROXY_VirtualKeyModelMaxBudgetLimiter
|
||||
|
||||
|
||||
def _make_user_api_key_auth(
|
||||
|
|
@ -235,3 +236,53 @@ async def test_no_max_budget_passes():
|
|||
|
||||
assert result is None
|
||||
mock_get_spend.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_log_success_event_uses_requested_model_budget():
|
||||
cache = DualCache()
|
||||
handler = _PROXY_VirtualKeyModelMaxBudgetLimiter(dual_cache=cache)
|
||||
|
||||
await handler.async_log_success_event(
|
||||
kwargs={
|
||||
"standard_logging_object": {
|
||||
"model_group": "claude-haiku-4-5-20251001",
|
||||
"model": "anthropic/claude-haiku-4-5-20251001",
|
||||
"response_cost": 5.5e-05,
|
||||
"metadata": {"user_api_key_hash": "hash-1"},
|
||||
},
|
||||
"litellm_params": {
|
||||
"metadata": {
|
||||
"user_api_key_model_max_budget": {
|
||||
"claude-haiku-4-5": {
|
||||
"budget_limit": 1e-4,
|
||||
"time_period": "1h",
|
||||
}
|
||||
},
|
||||
"litellm_client_requested_model": "claude-haiku-4-5",
|
||||
}
|
||||
},
|
||||
},
|
||||
response_obj=None,
|
||||
start_time=0,
|
||||
end_time=0,
|
||||
)
|
||||
|
||||
assert cache.in_memory_cache.cache_dict["virtual_key_spend:hash-1:claude-haiku-4-5:1h"] == 5.5e-05
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_fallback_model_within_budget_uses_alias_budget_fallbacks():
|
||||
handler = _PROXY_VirtualKeyModelMaxBudgetLimiter(dual_cache=DualCache())
|
||||
user_api_key_dict = UserAPIKeyAuth(
|
||||
api_key="sk-test",
|
||||
model_max_budget={},
|
||||
budget_fallbacks={"claude-haiku-4-5": ["claude-sonnet-4-5"]},
|
||||
)
|
||||
|
||||
result = await handler.get_fallback_model_within_budget(
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
model="anthropic/claude-haiku-4-5",
|
||||
)
|
||||
|
||||
assert result == "claude-sonnet-4-5"
|
||||
|
|
|
|||
|
|
@ -3895,6 +3895,7 @@ def test_update_model_if_key_alias_exists():
|
|||
_update_model_if_key_alias_exists(data=data, user_api_key_dict=user_api_key_dict)
|
||||
assert data["model"] == original_model # Should remain unchanged
|
||||
|
||||
|
||||
# Test case 3: Model is None
|
||||
data = {"model": None, "messages": [{"role": "user", "content": "Hello"}]}
|
||||
user_api_key_dict = UserAPIKeyAuth(
|
||||
|
|
@ -3934,6 +3935,49 @@ def test_update_model_if_key_alias_exists():
|
|||
assert data["model"] == original_model # Should remain unchanged
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_litellm_data_to_request_preserves_client_model_for_key_alias():
|
||||
request_mock = MagicMock(spec=Request)
|
||||
request_mock.url.path = "/chat/completions"
|
||||
request_mock.url = MagicMock()
|
||||
request_mock.url.__str__.return_value = "http://localhost/chat/completions"
|
||||
request_mock.method = "POST"
|
||||
request_mock.query_params = {}
|
||||
request_mock.headers = {"Content-Type": "application/json"}
|
||||
request_mock.client = MagicMock()
|
||||
request_mock.client.host = "127.0.0.1"
|
||||
|
||||
data = {"model": "claude-haiku-4-5"}
|
||||
user_api_key_dict = UserAPIKeyAuth(
|
||||
api_key="test-key",
|
||||
aliases={"claude-haiku-4-5": "claude-haiku-4-5-20251001"},
|
||||
)
|
||||
|
||||
updated_data = await add_litellm_data_to_request(
|
||||
data=data,
|
||||
request=request_mock,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
proxy_config=MagicMock(),
|
||||
general_settings={},
|
||||
version="test-version",
|
||||
)
|
||||
|
||||
assert updated_data["model"] == "claude-haiku-4-5-20251001"
|
||||
assert updated_data["metadata"]["litellm_client_requested_model"] == "claude-haiku-4-5"
|
||||
|
||||
data_without_alias = {"model": "claude-haiku-4-5"}
|
||||
updated_without_alias = await add_litellm_data_to_request(
|
||||
data=data_without_alias,
|
||||
request=request_mock,
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="test-key"),
|
||||
proxy_config=MagicMock(),
|
||||
general_settings={},
|
||||
version="test-version",
|
||||
)
|
||||
|
||||
assert "litellm_client_requested_model" not in updated_without_alias["metadata"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_embedding_header_forwarding_with_model_group():
|
||||
"""
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue