fix(proxy): enforce per-model key budgets across key and team model aliases

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
shivam 2026-09-12 01:26:51 +00:00
parent 50cd26cd9c
commit 22e2e39628
6 changed files with 206 additions and 22 deletions

View file

@ -225,14 +225,30 @@ async def _normalize_claude_model(
def _get_model_names_for_budget_checks(
model: str | list[str] | None,
aliases: Mapping[str, str] | None = None,
) -> list[str]:
if model is None:
return []
if isinstance(model, str):
return [model]
alias: Final = aliases.get(model) if isinstance(aliases, Mapping) else None
return [model, alias] if isinstance(alias, str) and alias != model else [model]
return model
def _get_model_names_for_token_budget_checks(
model: str | list[str] | None,
valid_token: UserAPIKeyAuth,
) -> list[str]:
return list(
dict.fromkeys(
(
*_get_model_names_for_budget_checks(model=model, aliases=valid_token.aliases),
*_get_model_names_for_budget_checks(model=model, aliases=valid_token.team_model_aliases),
)
)
)
class _KeyModelBudgetLimiter(Protocol):
async def is_key_within_model_budget(self, user_api_key_dict: UserAPIKeyAuth, model: str) -> bool: ...
@ -308,6 +324,7 @@ async def _check_key_model_budget_with_fallback(
valid_token: UserAPIKeyAuth,
model_max_budget_limiter: _KeyModelBudgetLimiter,
model_name: str,
requested_model: str,
request_data: dict,
request: Request,
llm_model_list: list | None = None,
@ -327,7 +344,7 @@ async def _check_key_model_budget_with_fallback(
regardless of whether they consume ``_read_request_body()``,
``request.json()``, or the path ``model`` parameter.
Fallback is only attempted when ``model_name`` matches the top-level
Fallback is only attempted when ``requested_model`` matches the top-level
``request_data["model"]``; models extracted from nested fields
(``session.model``, ``completion.model``, etc.) are not rewritable
and raise immediately.
@ -342,7 +359,7 @@ async def _check_key_model_budget_with_fallback(
model=model_name,
)
except litellm.BudgetExceededError as e:
if request_data.get("model") != model_name:
if request_data.get("model") != requested_model:
raise e
fallback_model: Final = await model_max_budget_limiter.get_fallback_model_within_budget(
user_api_key_dict=valid_token,
@ -1618,13 +1635,14 @@ async def _user_api_key_auth_builder(
await _check_user_model_budget(
valid_token=cast(UserAPIKeyAuth, valid_token),
model_max_budget_limiter=model_max_budget_limiter,
models=_get_model_names_for_budget_checks(
models=_get_model_names_for_token_budget_checks(
model=_get_model_from_request_context(
request_data=request_data,
route=route,
request=request,
llm_router=llm_router,
)
),
valid_token=valid_token,
),
)
@ -2130,7 +2148,10 @@ async def _user_api_key_auth_builder(
request=request,
llm_router=llm_router,
)
current_models = _get_model_names_for_budget_checks(model=current_model)
current_models = _get_model_names_for_token_budget_checks(
model=current_model,
valid_token=valid_token,
)
if (
max_budget_per_model is not None
@ -2146,6 +2167,7 @@ async def _user_api_key_auth_builder(
valid_token=valid_token,
model_max_budget_limiter=model_max_budget_limiter,
model_name=model_name,
requested_model=current_model if isinstance(current_model, str) else model_name,
request_data=request_data,
request=request,
llm_model_list=llm_model_list,
@ -2160,7 +2182,10 @@ async def _user_api_key_auth_builder(
request=request,
llm_router=llm_router,
)
current_models = _get_model_names_for_budget_checks(model=current_model)
current_models = _get_model_names_for_token_budget_checks(
model=current_model,
valid_token=valid_token,
)
# Check 5a. Internal user model_max_budget
if current_models:
@ -3298,7 +3323,7 @@ async def _run_post_custom_auth_checks(
request=request,
llm_router=llm_router,
)
current_models = _get_model_names_for_budget_checks(model=current_model)
current_models = _get_model_names_for_token_budget_checks(model=current_model, valid_token=valid_token)
# A zero-cost model cannot move any counter, so refusing it means refusing on
# spend some other model accrued. The JWT and virtual-key paths already skip
@ -3325,6 +3350,7 @@ async def _run_post_custom_auth_checks(
valid_token=valid_token,
model_max_budget_limiter=model_max_budget_limiter,
model_name=model_name,
requested_model=current_model if isinstance(current_model, str) else model_name,
request_data=request_data,
request=request,
llm_model_list=llm_model_list,
@ -3339,7 +3365,7 @@ async def _run_post_custom_auth_checks(
request=request,
llm_router=llm_router,
)
current_models = _get_model_names_for_budget_checks(model=current_model)
current_models = _get_model_names_for_token_budget_checks(model=current_model, valid_token=valid_token)
# 3b. Attach and check the internal user's model_max_budget.
# Custom auth builds its own token, so unlike the main path nothing has

View file

@ -245,19 +245,34 @@ def _as_spend(current_spend: object) -> float:
def _resolve_entity_model_budgets(
model: str,
models: tuple[str, ...],
entity_budgets: Iterable[tuple[Litellm_EntityType, str | None, object]],
) -> tuple[tuple[Litellm_EntityType, str, ResolvedModelBudget], ...]:
"""Drop the scopes that do not budget `model`, keeping only what can be incremented."""
"""Drop scopes that do not budget any request model, keeping only what can be incremented."""
return tuple(
(entity_type, entity_id, resolved)
for entity_type, entity_id, model_max_budget in entity_budgets
if entity_id is not None and isinstance(model_max_budget, Mapping) and model_max_budget
for resolved in (resolve_model_budget(model=model, model_max_budget=model_max_budget),)
for resolved in (_first_resolved(models=models, model_max_budget=model_max_budget),)
if resolved is not None and resolved.budget_config.budget_duration is not None
)
def _first_resolved(
models: tuple[str, ...],
model_max_budget: Mapping[str, object],
) -> ResolvedModelBudget | None:
return next(
(
resolved
for model in models
for resolved in (resolve_model_budget(model=model, model_max_budget=model_max_budget),)
if resolved is not None and resolved.budget_config.budget_duration is not None
),
None,
)
class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
"""
Handles budgets for model + virtual key
@ -298,7 +313,15 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
model: str,
) -> str | None:
budget_fallbacks: Final[dict[str, list[str]]] = user_api_key_dict.budget_fallbacks or {}
for fallback_model in budget_fallbacks.get(model, []):
fallback_models: Final = next(
(
budget_fallbacks[candidate]
for candidate in _budget_model_candidates(model)
if candidate in budget_fallbacks
),
(),
)
for fallback_model in fallback_models:
try:
await self.is_key_within_model_budget(user_api_key_dict=user_api_key_dict, model=fallback_model)
return fallback_model
@ -445,16 +468,12 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
_metadata: Final[dict] = _litellm_params.get("metadata", {}) or {}
payload_metadata: Final = standard_logging_payload.get("metadata") or {}
# Use model_group (the user-facing model alias, e.g. "gpt-4o") when
# available. The enforcement path receives the model name from
# request_data["model"] which is the model group alias, so the spend
# tracking cache key must resolve from the same name. Falling back to
# the deployment-level "model" field preserves behaviour for non-proxy
# or non-router deployments where model_group is None.
model: Final = standard_logging_payload.get("model_group") or standard_logging_payload.get("model")
if model is None:
return
requested_model: Final = _metadata.get("litellm_client_requested_model")
models: Final = tuple(dict.fromkeys(m for m in (model, requested_model) if isinstance(m, str)))
response_cost: Final[float] = standard_logging_payload.get("response_cost", 0)
entity_budgets: Final = (
(
@ -474,12 +493,12 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
),
)
resolved_budgets: Final = _resolve_entity_model_budgets(model=model, entity_budgets=entity_budgets)
resolved_budgets: Final = _resolve_entity_model_budgets(models=models, entity_budgets=entity_budgets)
if not resolved_budgets:
verbose_proxy_logger.debug(
"Not running _PROXY_VirtualKeyModelMaxBudgetLimiter.async_log_success_event: "
"no key, user or end-user model_max_budget covers model=%s",
model,
"no key, user or end-user model_max_budget covers models=%s",
models,
)
return

View file

@ -2439,6 +2439,9 @@ async def add_litellm_data_to_request(
user_api_key_dict=user_api_key_dict,
)
if isinstance(_pre_alias_model, str) and data.get("model") != _pre_alias_model:
data[_metadata_variable_name]["litellm_client_requested_model"] = _pre_alias_model # rebind-ok: metadata out-param
verbose_proxy_logger.debug("[PROXY] returned data from litellm_pre_call_utils: %s", data)
# Team/Project credential overrides from model_config

View file

@ -35,6 +35,7 @@ from litellm.proxy.auth.auth_checks import get_key_object, _cache_key_object
from litellm.proxy.auth.route_checks import RouteChecks
from litellm.proxy.auth.user_api_key_auth import (
_check_key_model_budget_with_fallback,
_get_model_names_for_budget_checks,
_ensure_litellm_received_at_on_request_state,
_ensure_parent_otel_span_on_request_state,
_PendingAutoRegister,
@ -50,6 +51,13 @@ from litellm.proxy.auth.user_api_key_auth import (
)
def test_get_model_names_for_budget_checks_includes_alias_target():
assert _get_model_names_for_budget_checks(
"claude-haiku-4-5",
aliases={"claude-haiku-4-5": "claude-haiku-4-5-20251001"},
) == ["claude-haiku-4-5", "claude-haiku-4-5-20251001"]
class _RoutingRequest:
def __init__(self, headers=None, query_params=None):
self.headers = headers or {}
@ -6399,6 +6407,7 @@ class TestCheckKeyModelBudgetWithFallback:
valid_token=valid_token,
model_max_budget_limiter=limiter,
model_name="gpt-4o",
requested_model="gpt-4o",
request_data=request_data,
request=request,
)
@ -6425,6 +6434,7 @@ class TestCheckKeyModelBudgetWithFallback:
valid_token=valid_token,
model_max_budget_limiter=limiter,
model_name="gpt-4o",
requested_model="gpt-4o",
request_data=request_data,
request=request,
)
@ -6439,6 +6449,31 @@ class TestCheckKeyModelBudgetWithFallback:
cached_keys, cached_body = request.scope["parsed_body"]
assert cached_body["model"] == "gpt-4o-mini"
@pytest.mark.asyncio
async def test_alias_target_budget_reroutes_original_request_to_fallback(self):
valid_token = UserAPIKeyAuth(
token="test-key",
budget_fallbacks={"claude-haiku-4-5-20251001": ["claude-sonnet-4-5"]},
)
limiter = AsyncMock()
limiter.is_key_within_model_budget.side_effect = litellm.BudgetExceededError(
current_cost=10, max_budget=5
)
limiter.get_fallback_model_within_budget.return_value = "claude-sonnet-4-5"
request_data = {"model": "claude-haiku-4-5"}
request = self._make_request()
await _check_key_model_budget_with_fallback(
valid_token=valid_token,
model_max_budget_limiter=limiter,
model_name="claude-haiku-4-5-20251001",
requested_model="claude-haiku-4-5",
request_data=request_data,
request=request,
)
assert request_data["model"] == "claude-sonnet-4-5"
@pytest.mark.asyncio
async def test_raises_when_every_fallback_also_exceeded(self):
valid_token = UserAPIKeyAuth(
@ -6456,6 +6491,7 @@ class TestCheckKeyModelBudgetWithFallback:
valid_token=valid_token,
model_max_budget_limiter=limiter,
model_name="gpt-4o",
requested_model="gpt-4o",
request_data=request_data,
request=request,
)
@ -6494,6 +6530,7 @@ class TestCheckKeyModelBudgetWithFallback:
valid_token=valid_token,
model_max_budget_limiter=limiter,
model_name="gpt-4o",
requested_model="gpt-4o",
request_data=request_data,
request=request,
llm_model_list=None,
@ -6528,6 +6565,7 @@ class TestCheckKeyModelBudgetWithFallback:
valid_token=valid_token,
model_max_budget_limiter=limiter,
model_name="gpt-4o",
requested_model="gpt-4o",
request_data=request_data,
request=request,
llm_model_list=None,
@ -6563,6 +6601,7 @@ class TestCheckKeyModelBudgetWithFallback:
valid_token=valid_token,
model_max_budget_limiter=limiter,
model_name="gpt-4o",
requested_model="gpt-4o",
request_data=request_data,
request=request,
llm_model_list=None,
@ -6599,6 +6638,7 @@ class TestCheckKeyModelBudgetWithFallback:
valid_token=valid_token,
model_max_budget_limiter=limiter,
model_name="gpt-4o",
requested_model="gpt-4o",
request_data=request_data,
request=request,
llm_model_list=None,
@ -6629,6 +6669,7 @@ class TestCheckKeyModelBudgetWithFallback:
valid_token=valid_token,
model_max_budget_limiter=limiter,
model_name="gpt-4o",
requested_model="gpt-4o",
request_data=request_data,
request=request,
)

View file

@ -19,6 +19,7 @@ from fastapi import HTTPException
from litellm.caching.caching import DualCache
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.hooks.max_budget_limiter import _PROXY_MaxBudgetLimiter
from litellm.proxy.hooks.model_max_budget_limiter import _PROXY_VirtualKeyModelMaxBudgetLimiter
def _make_user_api_key_auth(
@ -235,3 +236,53 @@ async def test_no_max_budget_passes():
assert result is None
mock_get_spend.assert_not_awaited()
@pytest.mark.asyncio
async def test_async_log_success_event_uses_requested_model_budget():
cache = DualCache()
handler = _PROXY_VirtualKeyModelMaxBudgetLimiter(dual_cache=cache)
await handler.async_log_success_event(
kwargs={
"standard_logging_object": {
"model_group": "claude-haiku-4-5-20251001",
"model": "anthropic/claude-haiku-4-5-20251001",
"response_cost": 5.5e-05,
"metadata": {"user_api_key_hash": "hash-1"},
},
"litellm_params": {
"metadata": {
"user_api_key_model_max_budget": {
"claude-haiku-4-5": {
"budget_limit": 1e-4,
"time_period": "1h",
}
},
"litellm_client_requested_model": "claude-haiku-4-5",
}
},
},
response_obj=None,
start_time=0,
end_time=0,
)
assert cache.in_memory_cache.cache_dict["virtual_key_spend:hash-1:claude-haiku-4-5:1h"] == 5.5e-05
@pytest.mark.asyncio
async def test_get_fallback_model_within_budget_uses_alias_budget_fallbacks():
handler = _PROXY_VirtualKeyModelMaxBudgetLimiter(dual_cache=DualCache())
user_api_key_dict = UserAPIKeyAuth(
api_key="sk-test",
model_max_budget={},
budget_fallbacks={"claude-haiku-4-5": ["claude-sonnet-4-5"]},
)
result = await handler.get_fallback_model_within_budget(
user_api_key_dict=user_api_key_dict,
model="anthropic/claude-haiku-4-5",
)
assert result == "claude-sonnet-4-5"

View file

@ -3895,6 +3895,7 @@ def test_update_model_if_key_alias_exists():
_update_model_if_key_alias_exists(data=data, user_api_key_dict=user_api_key_dict)
assert data["model"] == original_model # Should remain unchanged
# Test case 3: Model is None
data = {"model": None, "messages": [{"role": "user", "content": "Hello"}]}
user_api_key_dict = UserAPIKeyAuth(
@ -3934,6 +3935,49 @@ def test_update_model_if_key_alias_exists():
assert data["model"] == original_model # Should remain unchanged
@pytest.mark.asyncio
async def test_add_litellm_data_to_request_preserves_client_model_for_key_alias():
request_mock = MagicMock(spec=Request)
request_mock.url.path = "/chat/completions"
request_mock.url = MagicMock()
request_mock.url.__str__.return_value = "http://localhost/chat/completions"
request_mock.method = "POST"
request_mock.query_params = {}
request_mock.headers = {"Content-Type": "application/json"}
request_mock.client = MagicMock()
request_mock.client.host = "127.0.0.1"
data = {"model": "claude-haiku-4-5"}
user_api_key_dict = UserAPIKeyAuth(
api_key="test-key",
aliases={"claude-haiku-4-5": "claude-haiku-4-5-20251001"},
)
updated_data = await add_litellm_data_to_request(
data=data,
request=request_mock,
user_api_key_dict=user_api_key_dict,
proxy_config=MagicMock(),
general_settings={},
version="test-version",
)
assert updated_data["model"] == "claude-haiku-4-5-20251001"
assert updated_data["metadata"]["litellm_client_requested_model"] == "claude-haiku-4-5"
data_without_alias = {"model": "claude-haiku-4-5"}
updated_without_alias = await add_litellm_data_to_request(
data=data_without_alias,
request=request_mock,
user_api_key_dict=UserAPIKeyAuth(api_key="test-key"),
proxy_config=MagicMock(),
general_settings={},
version="test-version",
)
assert "litellm_client_requested_model" not in updated_without_alias["metadata"]
@pytest.mark.asyncio
async def test_embedding_header_forwarding_with_model_group():
"""