mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-09 03:18:44 +00:00
fix: make cache updates synchronous for budget enforcement
The budget enforcement was failing in tests because cache updates were fire-and-forget (asyncio.create_task), causing race conditions where subsequent requests would read stale spend data. Changes: 1. proxy_track_cost_callback.py: await update_cache() instead of create_task 2. proxy_server.py: await async_set_cache_pipeline() instead of create_task 3. auth_checks.py: prefer valid_token.team_member_spend (from fresh cache) over team_membership.spend (which may be stale) This ensures budget checks see the most recent spend values and properly enforce budget limits when requests come in quick succession. Fixes: test_users_in_team_budget, test_chat_completion_low_budget
This commit is contained in:
parent
d9da49bc35
commit
d0383412e8
3 changed files with 21 additions and 18 deletions
|
|
@ -2295,7 +2295,13 @@ async def _check_team_member_budget(
|
|||
and team_membership.litellm_budget_table.max_budget is not None
|
||||
):
|
||||
team_member_budget = team_membership.litellm_budget_table.max_budget
|
||||
team_member_spend = team_membership.spend or 0.0
|
||||
# Prefer valid_token.team_member_spend (from token cache) over team_membership.spend
|
||||
# The token cache is updated synchronously after each request, while the team membership
|
||||
# cache may have stale data since DB updates are batched
|
||||
if valid_token.team_member_spend is not None:
|
||||
team_member_spend = valid_token.team_member_spend
|
||||
else:
|
||||
team_member_spend = team_membership.spend or 0.0
|
||||
|
||||
if team_member_spend >= team_member_budget:
|
||||
raise litellm.BudgetExceededError(
|
||||
|
|
|
|||
|
|
@ -181,17 +181,15 @@ class _ProxyDBLogger(CustomLogger):
|
|||
org_id=org_id,
|
||||
)
|
||||
|
||||
# update cache
|
||||
asyncio.create_task(
|
||||
update_cache(
|
||||
token=user_api_key,
|
||||
user_id=user_id,
|
||||
end_user_id=end_user_id,
|
||||
response_cost=response_cost,
|
||||
team_id=team_id,
|
||||
parent_otel_span=parent_otel_span,
|
||||
tags=tags,
|
||||
)
|
||||
# update cache - await to ensure budget checks see updated spend
|
||||
await update_cache(
|
||||
token=user_api_key,
|
||||
user_id=user_id,
|
||||
end_user_id=end_user_id,
|
||||
response_cost=response_cost,
|
||||
team_id=team_id,
|
||||
parent_otel_span=parent_otel_span,
|
||||
tags=tags,
|
||||
)
|
||||
|
||||
await proxy_logging_obj.slack_alerting_instance.customer_spend_alert(
|
||||
|
|
|
|||
|
|
@ -1703,12 +1703,11 @@ async def update_cache( # noqa: PLR0915
|
|||
if tags is not None:
|
||||
await _update_tag_cache()
|
||||
|
||||
asyncio.create_task(
|
||||
user_api_key_cache.async_set_cache_pipeline(
|
||||
cache_list=values_to_update_in_cache,
|
||||
ttl=60,
|
||||
litellm_parent_otel_span=parent_otel_span,
|
||||
)
|
||||
# Await cache update to ensure budget checks see updated spend values
|
||||
await user_api_key_cache.async_set_cache_pipeline(
|
||||
cache_list=values_to_update_in_cache,
|
||||
ttl=60,
|
||||
litellm_parent_otel_span=parent_otel_span,
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue