From 22e2e3962812e6465601ace3cfe841176b3adae1 Mon Sep 17 00:00:00 2001 From: shivam Date: Sat, 12 Sep 2026 01:26:51 +0000 Subject: [PATCH] fix(proxy): enforce per-model key budgets across key and team model aliases Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/auth/user_api_key_auth.py | 44 ++++++++++++---- .../proxy/hooks/model_max_budget_limiter.py | 45 +++++++++++----- litellm/proxy/litellm_pre_call_utils.py | 3 ++ .../proxy/auth/test_user_api_key_auth.py | 41 +++++++++++++++ .../proxy/hooks/test_max_budget_limiter.py | 51 +++++++++++++++++++ .../proxy/test_litellm_pre_call_utils.py | 44 ++++++++++++++++ 6 files changed, 206 insertions(+), 22 deletions(-) diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 20ab9904f46..21281807f35 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -225,14 +225,30 @@ async def _normalize_claude_model( def _get_model_names_for_budget_checks( model: str | list[str] | None, + aliases: Mapping[str, str] | None = None, ) -> list[str]: if model is None: return [] if isinstance(model, str): - return [model] + alias: Final = aliases.get(model) if isinstance(aliases, Mapping) else None + return [model, alias] if isinstance(alias, str) and alias != model else [model] return model +def _get_model_names_for_token_budget_checks( + model: str | list[str] | None, + valid_token: UserAPIKeyAuth, +) -> list[str]: + return list( + dict.fromkeys( + ( + *_get_model_names_for_budget_checks(model=model, aliases=valid_token.aliases), + *_get_model_names_for_budget_checks(model=model, aliases=valid_token.team_model_aliases), + ) + ) + ) + + class _KeyModelBudgetLimiter(Protocol): async def is_key_within_model_budget(self, user_api_key_dict: UserAPIKeyAuth, model: str) -> bool: ... @@ -308,6 +324,7 @@ async def _check_key_model_budget_with_fallback( valid_token: UserAPIKeyAuth, model_max_budget_limiter: _KeyModelBudgetLimiter, model_name: str, + requested_model: str, request_data: dict, request: Request, llm_model_list: list | None = None, @@ -327,7 +344,7 @@ async def _check_key_model_budget_with_fallback( regardless of whether they consume ``_read_request_body()``, ``request.json()``, or the path ``model`` parameter. - Fallback is only attempted when ``model_name`` matches the top-level + Fallback is only attempted when ``requested_model`` matches the top-level ``request_data["model"]``; models extracted from nested fields (``session.model``, ``completion.model``, etc.) are not rewritable and raise immediately. @@ -342,7 +359,7 @@ async def _check_key_model_budget_with_fallback( model=model_name, ) except litellm.BudgetExceededError as e: - if request_data.get("model") != model_name: + if request_data.get("model") != requested_model: raise e fallback_model: Final = await model_max_budget_limiter.get_fallback_model_within_budget( user_api_key_dict=valid_token, @@ -1618,13 +1635,14 @@ async def _user_api_key_auth_builder( await _check_user_model_budget( valid_token=cast(UserAPIKeyAuth, valid_token), model_max_budget_limiter=model_max_budget_limiter, - models=_get_model_names_for_budget_checks( + models=_get_model_names_for_token_budget_checks( model=_get_model_from_request_context( request_data=request_data, route=route, request=request, llm_router=llm_router, - ) + ), + valid_token=valid_token, ), ) @@ -2130,7 +2148,10 @@ async def _user_api_key_auth_builder( request=request, llm_router=llm_router, ) - current_models = _get_model_names_for_budget_checks(model=current_model) + current_models = _get_model_names_for_token_budget_checks( + model=current_model, + valid_token=valid_token, + ) if ( max_budget_per_model is not None @@ -2146,6 +2167,7 @@ async def _user_api_key_auth_builder( valid_token=valid_token, model_max_budget_limiter=model_max_budget_limiter, model_name=model_name, + requested_model=current_model if isinstance(current_model, str) else model_name, request_data=request_data, request=request, llm_model_list=llm_model_list, @@ -2160,7 +2182,10 @@ async def _user_api_key_auth_builder( request=request, llm_router=llm_router, ) - current_models = _get_model_names_for_budget_checks(model=current_model) + current_models = _get_model_names_for_token_budget_checks( + model=current_model, + valid_token=valid_token, + ) # Check 5a. Internal user model_max_budget if current_models: @@ -3298,7 +3323,7 @@ async def _run_post_custom_auth_checks( request=request, llm_router=llm_router, ) - current_models = _get_model_names_for_budget_checks(model=current_model) + current_models = _get_model_names_for_token_budget_checks(model=current_model, valid_token=valid_token) # A zero-cost model cannot move any counter, so refusing it means refusing on # spend some other model accrued. The JWT and virtual-key paths already skip @@ -3325,6 +3350,7 @@ async def _run_post_custom_auth_checks( valid_token=valid_token, model_max_budget_limiter=model_max_budget_limiter, model_name=model_name, + requested_model=current_model if isinstance(current_model, str) else model_name, request_data=request_data, request=request, llm_model_list=llm_model_list, @@ -3339,7 +3365,7 @@ async def _run_post_custom_auth_checks( request=request, llm_router=llm_router, ) - current_models = _get_model_names_for_budget_checks(model=current_model) + current_models = _get_model_names_for_token_budget_checks(model=current_model, valid_token=valid_token) # 3b. Attach and check the internal user's model_max_budget. # Custom auth builds its own token, so unlike the main path nothing has diff --git a/litellm/proxy/hooks/model_max_budget_limiter.py b/litellm/proxy/hooks/model_max_budget_limiter.py index efaaab277a9..2a3e87f81f1 100644 --- a/litellm/proxy/hooks/model_max_budget_limiter.py +++ b/litellm/proxy/hooks/model_max_budget_limiter.py @@ -245,19 +245,34 @@ def _as_spend(current_spend: object) -> float: def _resolve_entity_model_budgets( - model: str, + models: tuple[str, ...], entity_budgets: Iterable[tuple[Litellm_EntityType, str | None, object]], ) -> tuple[tuple[Litellm_EntityType, str, ResolvedModelBudget], ...]: - """Drop the scopes that do not budget `model`, keeping only what can be incremented.""" + """Drop scopes that do not budget any request model, keeping only what can be incremented.""" return tuple( (entity_type, entity_id, resolved) for entity_type, entity_id, model_max_budget in entity_budgets if entity_id is not None and isinstance(model_max_budget, Mapping) and model_max_budget - for resolved in (resolve_model_budget(model=model, model_max_budget=model_max_budget),) + for resolved in (_first_resolved(models=models, model_max_budget=model_max_budget),) if resolved is not None and resolved.budget_config.budget_duration is not None ) +def _first_resolved( + models: tuple[str, ...], + model_max_budget: Mapping[str, object], +) -> ResolvedModelBudget | None: + return next( + ( + resolved + for model in models + for resolved in (resolve_model_budget(model=model, model_max_budget=model_max_budget),) + if resolved is not None and resolved.budget_config.budget_duration is not None + ), + None, + ) + + class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting): """ Handles budgets for model + virtual key @@ -298,7 +313,15 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting): model: str, ) -> str | None: budget_fallbacks: Final[dict[str, list[str]]] = user_api_key_dict.budget_fallbacks or {} - for fallback_model in budget_fallbacks.get(model, []): + fallback_models: Final = next( + ( + budget_fallbacks[candidate] + for candidate in _budget_model_candidates(model) + if candidate in budget_fallbacks + ), + (), + ) + for fallback_model in fallback_models: try: await self.is_key_within_model_budget(user_api_key_dict=user_api_key_dict, model=fallback_model) return fallback_model @@ -445,16 +468,12 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting): _metadata: Final[dict] = _litellm_params.get("metadata", {}) or {} payload_metadata: Final = standard_logging_payload.get("metadata") or {} - # Use model_group (the user-facing model alias, e.g. "gpt-4o") when - # available. The enforcement path receives the model name from - # request_data["model"] which is the model group alias, so the spend - # tracking cache key must resolve from the same name. Falling back to - # the deployment-level "model" field preserves behaviour for non-proxy - # or non-router deployments where model_group is None. model: Final = standard_logging_payload.get("model_group") or standard_logging_payload.get("model") if model is None: return + requested_model: Final = _metadata.get("litellm_client_requested_model") + models: Final = tuple(dict.fromkeys(m for m in (model, requested_model) if isinstance(m, str))) response_cost: Final[float] = standard_logging_payload.get("response_cost", 0) entity_budgets: Final = ( ( @@ -474,12 +493,12 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting): ), ) - resolved_budgets: Final = _resolve_entity_model_budgets(model=model, entity_budgets=entity_budgets) + resolved_budgets: Final = _resolve_entity_model_budgets(models=models, entity_budgets=entity_budgets) if not resolved_budgets: verbose_proxy_logger.debug( "Not running _PROXY_VirtualKeyModelMaxBudgetLimiter.async_log_success_event: " - "no key, user or end-user model_max_budget covers model=%s", - model, + "no key, user or end-user model_max_budget covers models=%s", + models, ) return diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 8f7b515c22a..4ec39d95653 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -2439,6 +2439,9 @@ async def add_litellm_data_to_request( user_api_key_dict=user_api_key_dict, ) + if isinstance(_pre_alias_model, str) and data.get("model") != _pre_alias_model: + data[_metadata_variable_name]["litellm_client_requested_model"] = _pre_alias_model # rebind-ok: metadata out-param + verbose_proxy_logger.debug("[PROXY] returned data from litellm_pre_call_utils: %s", data) # Team/Project credential overrides from model_config diff --git a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py index 6cce6d0316b..7a5f66e5c39 100644 --- a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py +++ b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py @@ -35,6 +35,7 @@ from litellm.proxy.auth.auth_checks import get_key_object, _cache_key_object from litellm.proxy.auth.route_checks import RouteChecks from litellm.proxy.auth.user_api_key_auth import ( _check_key_model_budget_with_fallback, + _get_model_names_for_budget_checks, _ensure_litellm_received_at_on_request_state, _ensure_parent_otel_span_on_request_state, _PendingAutoRegister, @@ -50,6 +51,13 @@ from litellm.proxy.auth.user_api_key_auth import ( ) +def test_get_model_names_for_budget_checks_includes_alias_target(): + assert _get_model_names_for_budget_checks( + "claude-haiku-4-5", + aliases={"claude-haiku-4-5": "claude-haiku-4-5-20251001"}, + ) == ["claude-haiku-4-5", "claude-haiku-4-5-20251001"] + + class _RoutingRequest: def __init__(self, headers=None, query_params=None): self.headers = headers or {} @@ -6399,6 +6407,7 @@ class TestCheckKeyModelBudgetWithFallback: valid_token=valid_token, model_max_budget_limiter=limiter, model_name="gpt-4o", + requested_model="gpt-4o", request_data=request_data, request=request, ) @@ -6425,6 +6434,7 @@ class TestCheckKeyModelBudgetWithFallback: valid_token=valid_token, model_max_budget_limiter=limiter, model_name="gpt-4o", + requested_model="gpt-4o", request_data=request_data, request=request, ) @@ -6439,6 +6449,31 @@ class TestCheckKeyModelBudgetWithFallback: cached_keys, cached_body = request.scope["parsed_body"] assert cached_body["model"] == "gpt-4o-mini" + @pytest.mark.asyncio + async def test_alias_target_budget_reroutes_original_request_to_fallback(self): + valid_token = UserAPIKeyAuth( + token="test-key", + budget_fallbacks={"claude-haiku-4-5-20251001": ["claude-sonnet-4-5"]}, + ) + limiter = AsyncMock() + limiter.is_key_within_model_budget.side_effect = litellm.BudgetExceededError( + current_cost=10, max_budget=5 + ) + limiter.get_fallback_model_within_budget.return_value = "claude-sonnet-4-5" + request_data = {"model": "claude-haiku-4-5"} + request = self._make_request() + + await _check_key_model_budget_with_fallback( + valid_token=valid_token, + model_max_budget_limiter=limiter, + model_name="claude-haiku-4-5-20251001", + requested_model="claude-haiku-4-5", + request_data=request_data, + request=request, + ) + + assert request_data["model"] == "claude-sonnet-4-5" + @pytest.mark.asyncio async def test_raises_when_every_fallback_also_exceeded(self): valid_token = UserAPIKeyAuth( @@ -6456,6 +6491,7 @@ class TestCheckKeyModelBudgetWithFallback: valid_token=valid_token, model_max_budget_limiter=limiter, model_name="gpt-4o", + requested_model="gpt-4o", request_data=request_data, request=request, ) @@ -6494,6 +6530,7 @@ class TestCheckKeyModelBudgetWithFallback: valid_token=valid_token, model_max_budget_limiter=limiter, model_name="gpt-4o", + requested_model="gpt-4o", request_data=request_data, request=request, llm_model_list=None, @@ -6528,6 +6565,7 @@ class TestCheckKeyModelBudgetWithFallback: valid_token=valid_token, model_max_budget_limiter=limiter, model_name="gpt-4o", + requested_model="gpt-4o", request_data=request_data, request=request, llm_model_list=None, @@ -6563,6 +6601,7 @@ class TestCheckKeyModelBudgetWithFallback: valid_token=valid_token, model_max_budget_limiter=limiter, model_name="gpt-4o", + requested_model="gpt-4o", request_data=request_data, request=request, llm_model_list=None, @@ -6599,6 +6638,7 @@ class TestCheckKeyModelBudgetWithFallback: valid_token=valid_token, model_max_budget_limiter=limiter, model_name="gpt-4o", + requested_model="gpt-4o", request_data=request_data, request=request, llm_model_list=None, @@ -6629,6 +6669,7 @@ class TestCheckKeyModelBudgetWithFallback: valid_token=valid_token, model_max_budget_limiter=limiter, model_name="gpt-4o", + requested_model="gpt-4o", request_data=request_data, request=request, ) diff --git a/tests/test_litellm/proxy/hooks/test_max_budget_limiter.py b/tests/test_litellm/proxy/hooks/test_max_budget_limiter.py index 71671966d1a..9fdc2e5875e 100644 --- a/tests/test_litellm/proxy/hooks/test_max_budget_limiter.py +++ b/tests/test_litellm/proxy/hooks/test_max_budget_limiter.py @@ -19,6 +19,7 @@ from fastapi import HTTPException from litellm.caching.caching import DualCache from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.hooks.max_budget_limiter import _PROXY_MaxBudgetLimiter +from litellm.proxy.hooks.model_max_budget_limiter import _PROXY_VirtualKeyModelMaxBudgetLimiter def _make_user_api_key_auth( @@ -235,3 +236,53 @@ async def test_no_max_budget_passes(): assert result is None mock_get_spend.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_async_log_success_event_uses_requested_model_budget(): + cache = DualCache() + handler = _PROXY_VirtualKeyModelMaxBudgetLimiter(dual_cache=cache) + + await handler.async_log_success_event( + kwargs={ + "standard_logging_object": { + "model_group": "claude-haiku-4-5-20251001", + "model": "anthropic/claude-haiku-4-5-20251001", + "response_cost": 5.5e-05, + "metadata": {"user_api_key_hash": "hash-1"}, + }, + "litellm_params": { + "metadata": { + "user_api_key_model_max_budget": { + "claude-haiku-4-5": { + "budget_limit": 1e-4, + "time_period": "1h", + } + }, + "litellm_client_requested_model": "claude-haiku-4-5", + } + }, + }, + response_obj=None, + start_time=0, + end_time=0, + ) + + assert cache.in_memory_cache.cache_dict["virtual_key_spend:hash-1:claude-haiku-4-5:1h"] == 5.5e-05 + + +@pytest.mark.asyncio +async def test_get_fallback_model_within_budget_uses_alias_budget_fallbacks(): + handler = _PROXY_VirtualKeyModelMaxBudgetLimiter(dual_cache=DualCache()) + user_api_key_dict = UserAPIKeyAuth( + api_key="sk-test", + model_max_budget={}, + budget_fallbacks={"claude-haiku-4-5": ["claude-sonnet-4-5"]}, + ) + + result = await handler.get_fallback_model_within_budget( + user_api_key_dict=user_api_key_dict, + model="anthropic/claude-haiku-4-5", + ) + + assert result == "claude-sonnet-4-5" diff --git a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py index 37b983d709a..bdbf563523e 100644 --- a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py +++ b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py @@ -3895,6 +3895,7 @@ def test_update_model_if_key_alias_exists(): _update_model_if_key_alias_exists(data=data, user_api_key_dict=user_api_key_dict) assert data["model"] == original_model # Should remain unchanged + # Test case 3: Model is None data = {"model": None, "messages": [{"role": "user", "content": "Hello"}]} user_api_key_dict = UserAPIKeyAuth( @@ -3934,6 +3935,49 @@ def test_update_model_if_key_alias_exists(): assert data["model"] == original_model # Should remain unchanged +@pytest.mark.asyncio +async def test_add_litellm_data_to_request_preserves_client_model_for_key_alias(): + request_mock = MagicMock(spec=Request) + request_mock.url.path = "/chat/completions" + request_mock.url = MagicMock() + request_mock.url.__str__.return_value = "http://localhost/chat/completions" + request_mock.method = "POST" + request_mock.query_params = {} + request_mock.headers = {"Content-Type": "application/json"} + request_mock.client = MagicMock() + request_mock.client.host = "127.0.0.1" + + data = {"model": "claude-haiku-4-5"} + user_api_key_dict = UserAPIKeyAuth( + api_key="test-key", + aliases={"claude-haiku-4-5": "claude-haiku-4-5-20251001"}, + ) + + updated_data = await add_litellm_data_to_request( + data=data, + request=request_mock, + user_api_key_dict=user_api_key_dict, + proxy_config=MagicMock(), + general_settings={}, + version="test-version", + ) + + assert updated_data["model"] == "claude-haiku-4-5-20251001" + assert updated_data["metadata"]["litellm_client_requested_model"] == "claude-haiku-4-5" + + data_without_alias = {"model": "claude-haiku-4-5"} + updated_without_alias = await add_litellm_data_to_request( + data=data_without_alias, + request=request_mock, + user_api_key_dict=UserAPIKeyAuth(api_key="test-key"), + proxy_config=MagicMock(), + general_settings={}, + version="test-version", + ) + + assert "litellm_client_requested_model" not in updated_without_alias["metadata"] + + @pytest.mark.asyncio async def test_embedding_header_forwarding_with_model_group(): """