mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(proxy): stop alias budget checks once a budget fallback is applied
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
300768a943
commit
8cdedaf3cf
2 changed files with 45 additions and 5 deletions
|
|
@ -329,7 +329,7 @@ async def _check_key_model_budget_with_fallback(
|
|||
request: Request,
|
||||
llm_model_list: list | None = None,
|
||||
llm_router: litellm.Router | None = None,
|
||||
) -> None:
|
||||
) -> bool:
|
||||
"""
|
||||
Enforce the key's per-model budget for `model_name`. If exceeded and the
|
||||
key has a `budget_fallbacks` chain configured for `model_name`, reroute
|
||||
|
|
@ -352,6 +352,9 @@ async def _check_key_model_budget_with_fallback(
|
|||
Raises:
|
||||
BudgetExceededError: if `model_name` is over budget and no configured
|
||||
fallback is within budget either (or the fallback is not authorized).
|
||||
|
||||
Returns:
|
||||
True when the request is rewritten to a fallback model, otherwise False.
|
||||
"""
|
||||
try:
|
||||
await model_max_budget_limiter.is_key_within_model_budget(
|
||||
|
|
@ -392,6 +395,8 @@ async def _check_key_model_budget_with_fallback(
|
|||
path_params: Final = request.scope.get("path_params")
|
||||
if isinstance(path_params, dict) and "model" in path_params:
|
||||
path_params["model"] = fallback_model
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _get_bearer_token_or_received_api_key(api_key: str) -> str:
|
||||
|
|
@ -2163,7 +2168,7 @@ async def _user_api_key_auth_builder(
|
|||
):
|
||||
## GET THE SPEND FOR THIS MODEL
|
||||
for model_name in current_models:
|
||||
await _check_key_model_budget_with_fallback(
|
||||
if await _check_key_model_budget_with_fallback(
|
||||
valid_token=valid_token,
|
||||
model_max_budget_limiter=model_max_budget_limiter,
|
||||
model_name=model_name,
|
||||
|
|
@ -2172,7 +2177,8 @@ async def _user_api_key_auth_builder(
|
|||
request=request,
|
||||
llm_model_list=llm_model_list,
|
||||
llm_router=llm_router,
|
||||
)
|
||||
):
|
||||
break
|
||||
|
||||
# Recompute after a potential budget-fallback rewrite so
|
||||
# the end-user check below validates the final model
|
||||
|
|
@ -3346,7 +3352,7 @@ async def _run_post_custom_auth_checks(
|
|||
and valid_token.token is not None
|
||||
):
|
||||
for model_name in current_models:
|
||||
await _check_key_model_budget_with_fallback(
|
||||
if await _check_key_model_budget_with_fallback(
|
||||
valid_token=valid_token,
|
||||
model_max_budget_limiter=model_max_budget_limiter,
|
||||
model_name=model_name,
|
||||
|
|
@ -3355,7 +3361,8 @@ async def _run_post_custom_auth_checks(
|
|||
request=request,
|
||||
llm_model_list=llm_model_list,
|
||||
llm_router=llm_router,
|
||||
)
|
||||
):
|
||||
break
|
||||
|
||||
# Recompute after a potential budget-fallback rewrite so
|
||||
# the end-user check below validates the final model
|
||||
|
|
|
|||
|
|
@ -6474,6 +6474,39 @@ class TestCheckKeyModelBudgetWithFallback:
|
|||
|
||||
assert request_data["model"] == "claude-sonnet-4-5"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_alias_budget_loop_stops_after_fallback(self):
|
||||
valid_token = UserAPIKeyAuth(
|
||||
token="test-key",
|
||||
aliases={"claude-haiku-4-5": "claude-haiku-4-5-20251001"},
|
||||
budget_fallbacks={"claude-haiku-4-5": ["claude-sonnet-4-5"]},
|
||||
)
|
||||
limiter = AsyncMock()
|
||||
limiter.is_key_within_model_budget.side_effect = litellm.BudgetExceededError(
|
||||
current_cost=10, max_budget=5
|
||||
)
|
||||
limiter.get_fallback_model_within_budget.return_value = "claude-sonnet-4-5"
|
||||
request_data = {"model": "claude-haiku-4-5"}
|
||||
request = self._make_request()
|
||||
candidates = ["claude-haiku-4-5", "claude-haiku-4-5-20251001"]
|
||||
|
||||
for model_name in candidates:
|
||||
if await _check_key_model_budget_with_fallback(
|
||||
valid_token=valid_token,
|
||||
model_max_budget_limiter=limiter,
|
||||
model_name=model_name,
|
||||
requested_model="claude-haiku-4-5",
|
||||
request_data=request_data,
|
||||
request=request,
|
||||
):
|
||||
break
|
||||
|
||||
assert request_data["model"] == "claude-sonnet-4-5"
|
||||
limiter.is_key_within_model_budget.assert_awaited_once_with(
|
||||
user_api_key_dict=valid_token,
|
||||
model="claude-haiku-4-5",
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_raises_when_every_fallback_also_exceeded(self):
|
||||
valid_token = UserAPIKeyAuth(
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue