fix(proxy): apply other spend counters before a window row read and let no-window notes follow the key cache settings

This commit is contained in:
mateo-berri 2026-11-02 13:30:08 -08:00
parent 26525a678a
commit 3416db7042
6 changed files with 202 additions and 114 deletions

View file

@ -71,6 +71,7 @@ class UserApiKeyCache(DualCache):
default_in_memory_ttl: float | None = None,
default_redis_ttl: float | None = None,
key_object_in_memory_cache: InMemoryCache | None = None,
budget_windows_absent_in_memory_cache: InMemoryCache | None = None,
) -> None:
super().__init__(
in_memory_cache=in_memory_cache,
@ -84,8 +85,10 @@ class UserApiKeyCache(DualCache):
default_in_memory_ttl=default_in_memory_ttl,
default_redis_ttl=default_redis_ttl,
)
self.budget_windows_absent_cache: Final = InMemoryCache(
default_ttl=DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL
self.budget_windows_absent_cache: Final = DualCache(
in_memory_cache=budget_windows_absent_in_memory_cache
or InMemoryCache(default_ttl=DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL),
default_in_memory_ttl=default_in_memory_ttl,
)
def in_memory_cache_for(self, key: str) -> InMemoryCache:
@ -101,10 +104,14 @@ class UserApiKeyCache(DualCache):
self.key_object_cache.update_cache_ttl(
default_in_memory_ttl=default_in_memory_ttl, default_redis_ttl=default_redis_ttl
)
self.budget_windows_absent_cache.update_cache_ttl(
default_in_memory_ttl=default_in_memory_ttl, default_redis_ttl=None
)
def update_in_memory_max_size(self, max_size: int | None) -> None:
super().update_in_memory_max_size(max_size)
self.key_object_cache.update_in_memory_max_size(max_size)
self.budget_windows_absent_cache.update_in_memory_max_size(max_size)
def attach_redis_cache(
self, redis_cache: RedisCache | None = None, *, default_redis_ttl: float | None = None

View file

@ -26,6 +26,7 @@ from collections.abc import (
Collection,
Container,
Iterable,
Iterator,
Mapping,
MutableMapping,
Sequence,
@ -3701,7 +3702,68 @@ async def _increment_spend_counters_batched(
cost: Final[float] = response_cost
async def _key_scope(key_token: str) -> tuple[PendingSpendIncrement | BaseException, ...]:
async def _window_increment(
entity_type: Litellm_EntityType, entity_id: str, window: object
) -> PendingSpendIncrement | None:
duration: Final = (
window["budget_duration"] if isinstance(window, dict) else getattr(window, "budget_duration", None)
)
window_reset_at: Final = (
window.get("reset_at") if isinstance(window, dict) else getattr(window, "reset_at", None)
)
window_counter: Final = f"spend:{entity_type.value}:{entity_id}:window:{duration}"
window_start: Final = get_budget_window_start(window)
pending_window: Final = (
await _prepare_window_spend_counter_increment(
counter_key=window_counter,
entity_type=entity_type.value.capitalize(),
entity_id=entity_id,
window_duration=duration,
window_start=window_start,
increment=cost,
)
if window_counter not in reserved_counter_keys
else None
)
await _enqueue_window_spend_row_update(
entity_type=entity_type,
entity_id=entity_id,
reset_at=window_reset_at,
window_duration=duration,
window_start=window_start,
increment=cost,
request_started_at=request_started_at,
)
return pending_window
async def _window_increments(
entity_type: Litellm_EntityType, entity_id: str, budget_limits: object
) -> tuple[PendingSpendIncrement | BaseException, ...]:
if not isinstance(budget_limits, list):
return ()
window_pending: Final = await asyncio.gather(
*(_window_increment(entity_type, entity_id, window) for window in budget_limits), return_exceptions=True
)
return tuple(item for item in window_pending if item is not None)
async def _cached_window_increments(
entity_type: Litellm_EntityType, entity_id: str, cache_key: str
) -> tuple[_SpendScopeItem, ...]:
cached: Final[object] = _UNTYPED_VALUE.validate_python(await user_api_key_cache.async_get_cache(key=cache_key))
if cached is None:
return (_UncachedBudgetWindows(entity_type=entity_type, entity_id=entity_id),)
return await _window_increments(entity_type, entity_id, _cached_budget_limits(cached))
async def _stored_window_increments(
uncached: _UncachedBudgetWindows,
) -> tuple[PendingSpendIncrement | BaseException, ...]:
return await _window_increments(
uncached.entity_type,
uncached.entity_id,
await _stored_budget_limits(uncached.entity_type, uncached.entity_id),
)
async def _key_scope(key_token: str) -> tuple[_SpendScopeItem, ...]:
# key_token arrives pre-hashed from metadata["user_api_key"] (auth flow
# hashes raw "sk-..." keys before they reach the callback). The
# startswith("sk-") check is a safety net matching update_cache —
@ -3724,54 +3786,9 @@ async def _increment_spend_counters_batched(
)
)
async def _key_window_increment(window: object) -> PendingSpendIncrement | None:
duration = (
window["budget_duration"] if isinstance(window, dict) else getattr(window, "budget_duration", None)
)
key_window_reset_at = (
window.get("reset_at") if isinstance(window, dict) else getattr(window, "reset_at", None)
)
key_window_counter: Final = f"spend:key:{hashed_token}:window:{duration}"
key_window_start = get_budget_window_start(window)
pending_window: Final = (
await _prepare_window_spend_counter_increment(
counter_key=key_window_counter,
entity_type="Key",
entity_id=hashed_token,
window_duration=duration,
window_start=key_window_start,
increment=cost,
)
if key_window_counter not in reserved_counter_keys
else None
)
await _enqueue_window_spend_row_update(
entity_type=Litellm_EntityType.KEY,
entity_id=hashed_token,
reset_at=key_window_reset_at,
window_duration=duration,
window_start=key_window_start,
increment=cost,
request_started_at=request_started_at,
)
return pending_window
return key_pending + await _cached_window_increments(Litellm_EntityType.KEY, hashed_token, hashed_token)
key_obj: Final[object] = _UNTYPED_VALUE.validate_python(
await user_api_key_cache.async_get_cache(key=hashed_token)
)
key_budget_limits: Final = (
_cached_budget_limits(key_obj)
if key_obj is not None
else await _stored_budget_limits(Litellm_EntityType.KEY, hashed_token)
)
if not isinstance(key_budget_limits, list):
return key_pending
window_pending: Final = await asyncio.gather(
*(_key_window_increment(window) for window in key_budget_limits), return_exceptions=True
)
return key_pending + tuple(item for item in window_pending if item is not None)
async def _team_scope(scope_team_id: str) -> tuple[PendingSpendIncrement | BaseException, ...]:
async def _team_scope(scope_team_id: str) -> tuple[_SpendScopeItem, ...]:
team_counter_key: Final = f"spend:team:{scope_team_id}"
team_pending: Final[tuple[PendingSpendIncrement, ...]] = (
()
@ -3785,52 +3802,9 @@ async def _increment_spend_counters_batched(
)
)
async def _team_window_increment(window: object) -> PendingSpendIncrement | None:
duration = (
window["budget_duration"] if isinstance(window, dict) else getattr(window, "budget_duration", None)
)
team_window_reset_at = (
window.get("reset_at") if isinstance(window, dict) else getattr(window, "reset_at", None)
)
team_window_counter: Final = f"spend:team:{scope_team_id}:window:{duration}"
team_window_start = get_budget_window_start(window)
pending_window: Final = (
await _prepare_window_spend_counter_increment(
counter_key=team_window_counter,
entity_type="Team",
entity_id=scope_team_id,
window_duration=duration,
window_start=team_window_start,
increment=cost,
)
if team_window_counter not in reserved_counter_keys
else None
)
await _enqueue_window_spend_row_update(
entity_type=Litellm_EntityType.TEAM,
entity_id=scope_team_id,
reset_at=team_window_reset_at,
window_duration=duration,
window_start=team_window_start,
increment=cost,
request_started_at=request_started_at,
)
return pending_window
team_obj: Final[object] = _UNTYPED_VALUE.validate_python(
await user_api_key_cache.async_get_cache(key=f"team_id:{scope_team_id}")
return team_pending + await _cached_window_increments(
Litellm_EntityType.TEAM, scope_team_id, f"team_id:{scope_team_id}"
)
team_budget_limits: Final = (
_cached_budget_limits(team_obj)
if team_obj is not None
else await _stored_budget_limits(Litellm_EntityType.TEAM, scope_team_id)
)
if not isinstance(team_budget_limits, list):
return team_pending
window_pending: Final = await asyncio.gather(
*(_team_window_increment(window) for window in team_budget_limits), return_exceptions=True
)
return team_pending + tuple(item for item in window_pending if item is not None)
async def _team_member_scope(
scope_user_id: str, scope_team_id: str
@ -3901,22 +3875,30 @@ async def _increment_spend_counters_batched(
# return_exceptions so a failing scope does not leave its siblings running
# as orphaned tasks that race the caller's reservation-counter invalidation;
# all scopes settle, then the first error propagates as before.
scope_results: Final = await asyncio.gather(*scope_coros, return_exceptions=True)
scope_errors: Final = tuple(
item
for scope in scope_results
for item in (scope if isinstance(scope, tuple) else (scope,))
if isinstance(item, BaseException)
scope_items: Final = tuple(_flatten_spend_scopes(await asyncio.gather(*scope_coros, return_exceptions=True)))
uncached_windows: Final = tuple(item for item in scope_items if isinstance(item, _UncachedBudgetWindows))
await _apply_spend_counter_increments(
pending=reservation_update.pending
+ tuple(item for item in scope_items if isinstance(item, PendingSpendIncrement))
)
pending: Final = tuple(
item
for scope in scope_results
if not isinstance(scope, BaseException)
for item in scope
if not isinstance(item, BaseException)
stored_window_items: Final = (
tuple(
_flatten_spend_scopes(
await asyncio.gather(
*(_stored_window_increments(uncached) for uncached in uncached_windows), return_exceptions=True
)
)
)
if uncached_windows
else ()
)
await _apply_spend_counter_increments(pending=reservation_update.pending + pending)
stored_window_pending: Final = tuple(
item for item in stored_window_items if isinstance(item, PendingSpendIncrement)
)
if stored_window_pending:
await _apply_spend_counter_increments(pending=stored_window_pending)
stamp_budget_reservation_actual_cost(budget_reservation=budget_reservation, actual_cost=response_cost)
scope_errors: Final = tuple(item for item in scope_items + stored_window_items if isinstance(item, BaseException))
if scope_errors:
raise scope_errors[0]
@ -3924,6 +3906,25 @@ async def _increment_spend_counters_batched(
budget_reservation["finalized"] = True
@dataclass(frozen=True, slots=True)
class _UncachedBudgetWindows:
entity_type: Litellm_EntityType
entity_id: str
_SpendScopeItem: TypeAlias = PendingSpendIncrement | BaseException | _UncachedBudgetWindows
def _flatten_spend_scopes(
scope_results: Sequence[tuple[_SpendScopeItem, ...] | BaseException],
) -> Iterator[_SpendScopeItem]:
for scope in scope_results:
if isinstance(scope, BaseException):
yield scope
continue
yield from scope
@dataclass(frozen=True, slots=True)
class _ReservationCounterUpdate:
"""The reserved counters the direct increment must skip, and the adjustments that settle them on the actual

View file

@ -1,6 +1,6 @@
import hashlib
import json
from typing import Any
from typing import Any, Final
import pytest
@ -11,6 +11,7 @@ from litellm.constants import DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.common_utils.user_api_key_cache import (
UserApiKeyCache,
budget_windows_absent_cache_key,
end_user_cache_key,
get_management_object_ttl,
is_user_key_cache_key,
@ -32,6 +33,17 @@ class CapturingInMemoryCache(InMemoryCache):
super().set_cache(key, value, **kwargs)
class _ManualClock:
def __init__(self) -> None:
self.seconds = 1000.0
def __call__(self) -> float:
return self.seconds
def advance(self, seconds: float) -> None:
self.seconds += seconds
class FakeRedisCache(RedisCache):
"""
In-memory fake that enforces the UserApiKeyCache Redis payload contract.
@ -432,6 +444,29 @@ class TestUserKeyObjectPartition:
assert first_key.token == tokens[0]
assert cache.get_cache(end_user_cache_key("u0")) == {"user_id": "u0"}
def test_update_in_memory_max_size_applies_to_no_window_notes(self):
cache = UserApiKeyCache()
cache.update_in_memory_max_size(1)
cache.budget_windows_absent_cache.set_cache(key=budget_windows_absent_cache_key("key", "first"), value=True)
cache.budget_windows_absent_cache.set_cache(key=budget_windows_absent_cache_key("key", "second"), value=True)
assert cache.budget_windows_absent_cache.get_cache(key=budget_windows_absent_cache_key("key", "first")) is None
assert cache.budget_windows_absent_cache.get_cache(key=budget_windows_absent_cache_key("key", "second")) is True
def test_update_cache_ttl_applies_to_no_window_notes(self):
clock: Final = _ManualClock()
cache: Final = UserApiKeyCache(
default_in_memory_ttl=60, budget_windows_absent_in_memory_cache=InMemoryCache(clock=clock)
)
cache.update_cache_ttl(default_in_memory_ttl=5, default_redis_ttl=5)
note: Final = budget_windows_absent_cache_key("key", "no-windows")
cache.budget_windows_absent_cache.set_cache(key=note, value=True)
clock.advance(6)
assert cache.budget_windows_absent_cache.get_cache(key=note) is None
def test_update_in_memory_max_size_none_resets_key_object_partition_to_default(self):
cache = UserApiKeyCache(key_object_in_memory_cache=InMemoryCache(max_size_in_memory=1))
cache.update_in_memory_max_size(None)

View file

@ -6,7 +6,7 @@ from contextlib import ExitStack
from datetime import datetime, timedelta, timezone
from types import SimpleNamespace
from typing import Final
from unittest.mock import AsyncMock, MagicMock, patch
from unittest.mock import AsyncMock, MagicMock, call, patch
from uuid import UUID, uuid4
import pytest
@ -51,7 +51,11 @@ from litellm.proxy.auth.auth_checks import (
)
from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth
from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import AuthCacheInvalidationSubscriber
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache, project_cache_key
from litellm.proxy.common_utils.user_api_key_cache import (
UserApiKeyCache,
budget_windows_absent_cache_key,
project_cache_key,
)
from litellm.proxy.db.db_lookup_gate import db_lookup_stall_tracker
from litellm.proxy.management_endpoints.key_management_endpoints import (
KeyProjectBindingError,
@ -8911,7 +8915,10 @@ async def test_delete_cache_key_object_broadcasts_invalidation(monkeypatch):
# Real, observable state: the cache object itself no longer holds the entry.
assert real_user_api_key_cache.get_cache(key="hashed-broadcast-key") is None
mock_publish.assert_any_await(cache_key="hashed-broadcast-key")
assert mock_publish.await_args_list == [
call(cache_key="hashed-broadcast-key"),
call(cache_key=budget_windows_absent_cache_key("key", "hashed-broadcast-key")),
]
@pytest.mark.asyncio

View file

@ -2684,7 +2684,7 @@ async def test_team_block_unblock_reach_warmed_auth_on_every_worker(
from litellm.proxy._types import BlockTeamRequest
from litellm.proxy.auth.auth_checks import get_team_object
from litellm.proxy.auth.team_grants import team_model_aliases
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache, budget_windows_absent_cache_key
from litellm.proxy.management_endpoints import team_endpoints
team_id: Final = "team-kill-switch"
@ -2721,7 +2721,11 @@ async def test_team_block_unblock_reach_warmed_auth_on_every_worker(
seen_by_auth: Final = await get_team_object(team_id=team_id, prisma_client=prisma_client, user_api_key_cache=cache)
assert seen_by_auth.blocked is blocked
assert team_model_aliases(seen_by_auth) == {"fast": "gpt-4o"}
assert redis.broadcast_keys[:2] == [f"team_id:{team_id}", "team_alias:kill-switch"]
assert redis.broadcast_keys == [
f"team_id:{team_id}",
"team_alias:kill-switch",
budget_windows_absent_cache_key("team", team_id),
]
assert (response.blocked, response.litellm_model_table, response.object_permission) == (blocked, None, None)

View file

@ -14485,6 +14485,40 @@ async def test_late_requests_from_many_keys_with_windows_do_not_push_out_a_known
assert no_window_key_reads() == reads_after_the_first_request
@pytest.mark.asyncio
async def test_a_slow_window_row_read_does_not_hold_back_the_key_spend_counter():
import litellm.proxy.proxy_server as ps
from litellm.proxy.proxy_server import increment_spend_counters
prisma_client: Final = _prisma_with_rows()
read_started: Final = asyncio.Event()
finish_read: Final = asyncio.Event()
async def slow_row_read(where: Mapping[str, str]) -> _StoredRow:
read_started.set()
await finish_read.wait()
return _StoredRow([_window("7d", 5.0)])
prisma_client.db.litellm_verificationtoken.find_unique = slow_row_read
with _window_spend_enqueue_env({}, prisma_client=prisma_client) as queue:
ps.spend_counter_cache.set_cache(key="spend:key:hashed-token", value=0.0)
ps.spend_counter_cache.set_cache(key="spend:key:hashed-token:window:7d", value=1.0)
charge: Final = asyncio.create_task(
increment_spend_counters(token="hashed-token", team_id=None, user_id=None, response_cost=0.25)
)
assert await asyncio.wait_for(read_started.wait(), timeout=5)
key_spend_while_the_row_read_waits: Final = ps.spend_counter_cache.get_cache(key="spend:key:hashed-token")
finish_read.set()
await charge
window_spend: Final = ps.spend_counter_cache.get_cache(key="spend:key:hashed-token:window:7d")
charged: Final = await _drain(queue)
assert key_spend_while_the_row_read_waits == pytest.approx(0.25)
assert window_spend == pytest.approx(1.25)
assert _charged_windows(charged) == [("hashed-token", "7d", pytest.approx(0.25))]
@pytest.mark.asyncio
async def test_a_late_request_that_misses_the_updated_team_charges_windows_the_team_did_not_have_before():
"""Another worker's /team/update only evicts this worker's cached team, so its next late cost reads the row."""