mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-15 09:12:18 +00:00
fix(proxy): apply other spend counters before a window row read and let no-window notes follow the key cache settings
This commit is contained in:
parent
26525a678a
commit
3416db7042
6 changed files with 202 additions and 114 deletions
|
|
@ -71,6 +71,7 @@ class UserApiKeyCache(DualCache):
|
|||
default_in_memory_ttl: float | None = None,
|
||||
default_redis_ttl: float | None = None,
|
||||
key_object_in_memory_cache: InMemoryCache | None = None,
|
||||
budget_windows_absent_in_memory_cache: InMemoryCache | None = None,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
in_memory_cache=in_memory_cache,
|
||||
|
|
@ -84,8 +85,10 @@ class UserApiKeyCache(DualCache):
|
|||
default_in_memory_ttl=default_in_memory_ttl,
|
||||
default_redis_ttl=default_redis_ttl,
|
||||
)
|
||||
self.budget_windows_absent_cache: Final = InMemoryCache(
|
||||
default_ttl=DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL
|
||||
self.budget_windows_absent_cache: Final = DualCache(
|
||||
in_memory_cache=budget_windows_absent_in_memory_cache
|
||||
or InMemoryCache(default_ttl=DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL),
|
||||
default_in_memory_ttl=default_in_memory_ttl,
|
||||
)
|
||||
|
||||
def in_memory_cache_for(self, key: str) -> InMemoryCache:
|
||||
|
|
@ -101,10 +104,14 @@ class UserApiKeyCache(DualCache):
|
|||
self.key_object_cache.update_cache_ttl(
|
||||
default_in_memory_ttl=default_in_memory_ttl, default_redis_ttl=default_redis_ttl
|
||||
)
|
||||
self.budget_windows_absent_cache.update_cache_ttl(
|
||||
default_in_memory_ttl=default_in_memory_ttl, default_redis_ttl=None
|
||||
)
|
||||
|
||||
def update_in_memory_max_size(self, max_size: int | None) -> None:
|
||||
super().update_in_memory_max_size(max_size)
|
||||
self.key_object_cache.update_in_memory_max_size(max_size)
|
||||
self.budget_windows_absent_cache.update_in_memory_max_size(max_size)
|
||||
|
||||
def attach_redis_cache(
|
||||
self, redis_cache: RedisCache | None = None, *, default_redis_ttl: float | None = None
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ from collections.abc import (
|
|||
Collection,
|
||||
Container,
|
||||
Iterable,
|
||||
Iterator,
|
||||
Mapping,
|
||||
MutableMapping,
|
||||
Sequence,
|
||||
|
|
@ -3701,7 +3702,68 @@ async def _increment_spend_counters_batched(
|
|||
|
||||
cost: Final[float] = response_cost
|
||||
|
||||
async def _key_scope(key_token: str) -> tuple[PendingSpendIncrement | BaseException, ...]:
|
||||
async def _window_increment(
|
||||
entity_type: Litellm_EntityType, entity_id: str, window: object
|
||||
) -> PendingSpendIncrement | None:
|
||||
duration: Final = (
|
||||
window["budget_duration"] if isinstance(window, dict) else getattr(window, "budget_duration", None)
|
||||
)
|
||||
window_reset_at: Final = (
|
||||
window.get("reset_at") if isinstance(window, dict) else getattr(window, "reset_at", None)
|
||||
)
|
||||
window_counter: Final = f"spend:{entity_type.value}:{entity_id}:window:{duration}"
|
||||
window_start: Final = get_budget_window_start(window)
|
||||
pending_window: Final = (
|
||||
await _prepare_window_spend_counter_increment(
|
||||
counter_key=window_counter,
|
||||
entity_type=entity_type.value.capitalize(),
|
||||
entity_id=entity_id,
|
||||
window_duration=duration,
|
||||
window_start=window_start,
|
||||
increment=cost,
|
||||
)
|
||||
if window_counter not in reserved_counter_keys
|
||||
else None
|
||||
)
|
||||
await _enqueue_window_spend_row_update(
|
||||
entity_type=entity_type,
|
||||
entity_id=entity_id,
|
||||
reset_at=window_reset_at,
|
||||
window_duration=duration,
|
||||
window_start=window_start,
|
||||
increment=cost,
|
||||
request_started_at=request_started_at,
|
||||
)
|
||||
return pending_window
|
||||
|
||||
async def _window_increments(
|
||||
entity_type: Litellm_EntityType, entity_id: str, budget_limits: object
|
||||
) -> tuple[PendingSpendIncrement | BaseException, ...]:
|
||||
if not isinstance(budget_limits, list):
|
||||
return ()
|
||||
window_pending: Final = await asyncio.gather(
|
||||
*(_window_increment(entity_type, entity_id, window) for window in budget_limits), return_exceptions=True
|
||||
)
|
||||
return tuple(item for item in window_pending if item is not None)
|
||||
|
||||
async def _cached_window_increments(
|
||||
entity_type: Litellm_EntityType, entity_id: str, cache_key: str
|
||||
) -> tuple[_SpendScopeItem, ...]:
|
||||
cached: Final[object] = _UNTYPED_VALUE.validate_python(await user_api_key_cache.async_get_cache(key=cache_key))
|
||||
if cached is None:
|
||||
return (_UncachedBudgetWindows(entity_type=entity_type, entity_id=entity_id),)
|
||||
return await _window_increments(entity_type, entity_id, _cached_budget_limits(cached))
|
||||
|
||||
async def _stored_window_increments(
|
||||
uncached: _UncachedBudgetWindows,
|
||||
) -> tuple[PendingSpendIncrement | BaseException, ...]:
|
||||
return await _window_increments(
|
||||
uncached.entity_type,
|
||||
uncached.entity_id,
|
||||
await _stored_budget_limits(uncached.entity_type, uncached.entity_id),
|
||||
)
|
||||
|
||||
async def _key_scope(key_token: str) -> tuple[_SpendScopeItem, ...]:
|
||||
# key_token arrives pre-hashed from metadata["user_api_key"] (auth flow
|
||||
# hashes raw "sk-..." keys before they reach the callback). The
|
||||
# startswith("sk-") check is a safety net matching update_cache —
|
||||
|
|
@ -3724,54 +3786,9 @@ async def _increment_spend_counters_batched(
|
|||
)
|
||||
)
|
||||
|
||||
async def _key_window_increment(window: object) -> PendingSpendIncrement | None:
|
||||
duration = (
|
||||
window["budget_duration"] if isinstance(window, dict) else getattr(window, "budget_duration", None)
|
||||
)
|
||||
key_window_reset_at = (
|
||||
window.get("reset_at") if isinstance(window, dict) else getattr(window, "reset_at", None)
|
||||
)
|
||||
key_window_counter: Final = f"spend:key:{hashed_token}:window:{duration}"
|
||||
key_window_start = get_budget_window_start(window)
|
||||
pending_window: Final = (
|
||||
await _prepare_window_spend_counter_increment(
|
||||
counter_key=key_window_counter,
|
||||
entity_type="Key",
|
||||
entity_id=hashed_token,
|
||||
window_duration=duration,
|
||||
window_start=key_window_start,
|
||||
increment=cost,
|
||||
)
|
||||
if key_window_counter not in reserved_counter_keys
|
||||
else None
|
||||
)
|
||||
await _enqueue_window_spend_row_update(
|
||||
entity_type=Litellm_EntityType.KEY,
|
||||
entity_id=hashed_token,
|
||||
reset_at=key_window_reset_at,
|
||||
window_duration=duration,
|
||||
window_start=key_window_start,
|
||||
increment=cost,
|
||||
request_started_at=request_started_at,
|
||||
)
|
||||
return pending_window
|
||||
return key_pending + await _cached_window_increments(Litellm_EntityType.KEY, hashed_token, hashed_token)
|
||||
|
||||
key_obj: Final[object] = _UNTYPED_VALUE.validate_python(
|
||||
await user_api_key_cache.async_get_cache(key=hashed_token)
|
||||
)
|
||||
key_budget_limits: Final = (
|
||||
_cached_budget_limits(key_obj)
|
||||
if key_obj is not None
|
||||
else await _stored_budget_limits(Litellm_EntityType.KEY, hashed_token)
|
||||
)
|
||||
if not isinstance(key_budget_limits, list):
|
||||
return key_pending
|
||||
window_pending: Final = await asyncio.gather(
|
||||
*(_key_window_increment(window) for window in key_budget_limits), return_exceptions=True
|
||||
)
|
||||
return key_pending + tuple(item for item in window_pending if item is not None)
|
||||
|
||||
async def _team_scope(scope_team_id: str) -> tuple[PendingSpendIncrement | BaseException, ...]:
|
||||
async def _team_scope(scope_team_id: str) -> tuple[_SpendScopeItem, ...]:
|
||||
team_counter_key: Final = f"spend:team:{scope_team_id}"
|
||||
team_pending: Final[tuple[PendingSpendIncrement, ...]] = (
|
||||
()
|
||||
|
|
@ -3785,52 +3802,9 @@ async def _increment_spend_counters_batched(
|
|||
)
|
||||
)
|
||||
|
||||
async def _team_window_increment(window: object) -> PendingSpendIncrement | None:
|
||||
duration = (
|
||||
window["budget_duration"] if isinstance(window, dict) else getattr(window, "budget_duration", None)
|
||||
)
|
||||
team_window_reset_at = (
|
||||
window.get("reset_at") if isinstance(window, dict) else getattr(window, "reset_at", None)
|
||||
)
|
||||
team_window_counter: Final = f"spend:team:{scope_team_id}:window:{duration}"
|
||||
team_window_start = get_budget_window_start(window)
|
||||
pending_window: Final = (
|
||||
await _prepare_window_spend_counter_increment(
|
||||
counter_key=team_window_counter,
|
||||
entity_type="Team",
|
||||
entity_id=scope_team_id,
|
||||
window_duration=duration,
|
||||
window_start=team_window_start,
|
||||
increment=cost,
|
||||
)
|
||||
if team_window_counter not in reserved_counter_keys
|
||||
else None
|
||||
)
|
||||
await _enqueue_window_spend_row_update(
|
||||
entity_type=Litellm_EntityType.TEAM,
|
||||
entity_id=scope_team_id,
|
||||
reset_at=team_window_reset_at,
|
||||
window_duration=duration,
|
||||
window_start=team_window_start,
|
||||
increment=cost,
|
||||
request_started_at=request_started_at,
|
||||
)
|
||||
return pending_window
|
||||
|
||||
team_obj: Final[object] = _UNTYPED_VALUE.validate_python(
|
||||
await user_api_key_cache.async_get_cache(key=f"team_id:{scope_team_id}")
|
||||
return team_pending + await _cached_window_increments(
|
||||
Litellm_EntityType.TEAM, scope_team_id, f"team_id:{scope_team_id}"
|
||||
)
|
||||
team_budget_limits: Final = (
|
||||
_cached_budget_limits(team_obj)
|
||||
if team_obj is not None
|
||||
else await _stored_budget_limits(Litellm_EntityType.TEAM, scope_team_id)
|
||||
)
|
||||
if not isinstance(team_budget_limits, list):
|
||||
return team_pending
|
||||
window_pending: Final = await asyncio.gather(
|
||||
*(_team_window_increment(window) for window in team_budget_limits), return_exceptions=True
|
||||
)
|
||||
return team_pending + tuple(item for item in window_pending if item is not None)
|
||||
|
||||
async def _team_member_scope(
|
||||
scope_user_id: str, scope_team_id: str
|
||||
|
|
@ -3901,22 +3875,30 @@ async def _increment_spend_counters_batched(
|
|||
# return_exceptions so a failing scope does not leave its siblings running
|
||||
# as orphaned tasks that race the caller's reservation-counter invalidation;
|
||||
# all scopes settle, then the first error propagates as before.
|
||||
scope_results: Final = await asyncio.gather(*scope_coros, return_exceptions=True)
|
||||
scope_errors: Final = tuple(
|
||||
item
|
||||
for scope in scope_results
|
||||
for item in (scope if isinstance(scope, tuple) else (scope,))
|
||||
if isinstance(item, BaseException)
|
||||
scope_items: Final = tuple(_flatten_spend_scopes(await asyncio.gather(*scope_coros, return_exceptions=True)))
|
||||
uncached_windows: Final = tuple(item for item in scope_items if isinstance(item, _UncachedBudgetWindows))
|
||||
await _apply_spend_counter_increments(
|
||||
pending=reservation_update.pending
|
||||
+ tuple(item for item in scope_items if isinstance(item, PendingSpendIncrement))
|
||||
)
|
||||
pending: Final = tuple(
|
||||
item
|
||||
for scope in scope_results
|
||||
if not isinstance(scope, BaseException)
|
||||
for item in scope
|
||||
if not isinstance(item, BaseException)
|
||||
stored_window_items: Final = (
|
||||
tuple(
|
||||
_flatten_spend_scopes(
|
||||
await asyncio.gather(
|
||||
*(_stored_window_increments(uncached) for uncached in uncached_windows), return_exceptions=True
|
||||
)
|
||||
)
|
||||
)
|
||||
if uncached_windows
|
||||
else ()
|
||||
)
|
||||
await _apply_spend_counter_increments(pending=reservation_update.pending + pending)
|
||||
stored_window_pending: Final = tuple(
|
||||
item for item in stored_window_items if isinstance(item, PendingSpendIncrement)
|
||||
)
|
||||
if stored_window_pending:
|
||||
await _apply_spend_counter_increments(pending=stored_window_pending)
|
||||
stamp_budget_reservation_actual_cost(budget_reservation=budget_reservation, actual_cost=response_cost)
|
||||
scope_errors: Final = tuple(item for item in scope_items + stored_window_items if isinstance(item, BaseException))
|
||||
if scope_errors:
|
||||
raise scope_errors[0]
|
||||
|
||||
|
|
@ -3924,6 +3906,25 @@ async def _increment_spend_counters_batched(
|
|||
budget_reservation["finalized"] = True
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _UncachedBudgetWindows:
|
||||
entity_type: Litellm_EntityType
|
||||
entity_id: str
|
||||
|
||||
|
||||
_SpendScopeItem: TypeAlias = PendingSpendIncrement | BaseException | _UncachedBudgetWindows
|
||||
|
||||
|
||||
def _flatten_spend_scopes(
|
||||
scope_results: Sequence[tuple[_SpendScopeItem, ...] | BaseException],
|
||||
) -> Iterator[_SpendScopeItem]:
|
||||
for scope in scope_results:
|
||||
if isinstance(scope, BaseException):
|
||||
yield scope
|
||||
continue
|
||||
yield from scope
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _ReservationCounterUpdate:
|
||||
"""The reserved counters the direct increment must skip, and the adjustments that settle them on the actual
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import hashlib
|
||||
import json
|
||||
from typing import Any
|
||||
from typing import Any, Final
|
||||
|
||||
import pytest
|
||||
|
||||
|
|
@ -11,6 +11,7 @@ from litellm.constants import DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL
|
|||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.common_utils.user_api_key_cache import (
|
||||
UserApiKeyCache,
|
||||
budget_windows_absent_cache_key,
|
||||
end_user_cache_key,
|
||||
get_management_object_ttl,
|
||||
is_user_key_cache_key,
|
||||
|
|
@ -32,6 +33,17 @@ class CapturingInMemoryCache(InMemoryCache):
|
|||
super().set_cache(key, value, **kwargs)
|
||||
|
||||
|
||||
class _ManualClock:
|
||||
def __init__(self) -> None:
|
||||
self.seconds = 1000.0
|
||||
|
||||
def __call__(self) -> float:
|
||||
return self.seconds
|
||||
|
||||
def advance(self, seconds: float) -> None:
|
||||
self.seconds += seconds
|
||||
|
||||
|
||||
class FakeRedisCache(RedisCache):
|
||||
"""
|
||||
In-memory fake that enforces the UserApiKeyCache Redis payload contract.
|
||||
|
|
@ -432,6 +444,29 @@ class TestUserKeyObjectPartition:
|
|||
assert first_key.token == tokens[0]
|
||||
assert cache.get_cache(end_user_cache_key("u0")) == {"user_id": "u0"}
|
||||
|
||||
def test_update_in_memory_max_size_applies_to_no_window_notes(self):
|
||||
cache = UserApiKeyCache()
|
||||
cache.update_in_memory_max_size(1)
|
||||
|
||||
cache.budget_windows_absent_cache.set_cache(key=budget_windows_absent_cache_key("key", "first"), value=True)
|
||||
cache.budget_windows_absent_cache.set_cache(key=budget_windows_absent_cache_key("key", "second"), value=True)
|
||||
|
||||
assert cache.budget_windows_absent_cache.get_cache(key=budget_windows_absent_cache_key("key", "first")) is None
|
||||
assert cache.budget_windows_absent_cache.get_cache(key=budget_windows_absent_cache_key("key", "second")) is True
|
||||
|
||||
def test_update_cache_ttl_applies_to_no_window_notes(self):
|
||||
clock: Final = _ManualClock()
|
||||
cache: Final = UserApiKeyCache(
|
||||
default_in_memory_ttl=60, budget_windows_absent_in_memory_cache=InMemoryCache(clock=clock)
|
||||
)
|
||||
cache.update_cache_ttl(default_in_memory_ttl=5, default_redis_ttl=5)
|
||||
note: Final = budget_windows_absent_cache_key("key", "no-windows")
|
||||
|
||||
cache.budget_windows_absent_cache.set_cache(key=note, value=True)
|
||||
clock.advance(6)
|
||||
|
||||
assert cache.budget_windows_absent_cache.get_cache(key=note) is None
|
||||
|
||||
def test_update_in_memory_max_size_none_resets_key_object_partition_to_default(self):
|
||||
cache = UserApiKeyCache(key_object_in_memory_cache=InMemoryCache(max_size_in_memory=1))
|
||||
cache.update_in_memory_max_size(None)
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ from contextlib import ExitStack
|
|||
from datetime import datetime, timedelta, timezone
|
||||
from types import SimpleNamespace
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
from unittest.mock import AsyncMock, MagicMock, call, patch
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
import pytest
|
||||
|
|
@ -51,7 +51,11 @@ from litellm.proxy.auth.auth_checks import (
|
|||
)
|
||||
from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth
|
||||
from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import AuthCacheInvalidationSubscriber
|
||||
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache, project_cache_key
|
||||
from litellm.proxy.common_utils.user_api_key_cache import (
|
||||
UserApiKeyCache,
|
||||
budget_windows_absent_cache_key,
|
||||
project_cache_key,
|
||||
)
|
||||
from litellm.proxy.db.db_lookup_gate import db_lookup_stall_tracker
|
||||
from litellm.proxy.management_endpoints.key_management_endpoints import (
|
||||
KeyProjectBindingError,
|
||||
|
|
@ -8911,7 +8915,10 @@ async def test_delete_cache_key_object_broadcasts_invalidation(monkeypatch):
|
|||
|
||||
# Real, observable state: the cache object itself no longer holds the entry.
|
||||
assert real_user_api_key_cache.get_cache(key="hashed-broadcast-key") is None
|
||||
mock_publish.assert_any_await(cache_key="hashed-broadcast-key")
|
||||
assert mock_publish.await_args_list == [
|
||||
call(cache_key="hashed-broadcast-key"),
|
||||
call(cache_key=budget_windows_absent_cache_key("key", "hashed-broadcast-key")),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
|
|
@ -2684,7 +2684,7 @@ async def test_team_block_unblock_reach_warmed_auth_on_every_worker(
|
|||
from litellm.proxy._types import BlockTeamRequest
|
||||
from litellm.proxy.auth.auth_checks import get_team_object
|
||||
from litellm.proxy.auth.team_grants import team_model_aliases
|
||||
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
|
||||
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache, budget_windows_absent_cache_key
|
||||
from litellm.proxy.management_endpoints import team_endpoints
|
||||
|
||||
team_id: Final = "team-kill-switch"
|
||||
|
|
@ -2721,7 +2721,11 @@ async def test_team_block_unblock_reach_warmed_auth_on_every_worker(
|
|||
seen_by_auth: Final = await get_team_object(team_id=team_id, prisma_client=prisma_client, user_api_key_cache=cache)
|
||||
assert seen_by_auth.blocked is blocked
|
||||
assert team_model_aliases(seen_by_auth) == {"fast": "gpt-4o"}
|
||||
assert redis.broadcast_keys[:2] == [f"team_id:{team_id}", "team_alias:kill-switch"]
|
||||
assert redis.broadcast_keys == [
|
||||
f"team_id:{team_id}",
|
||||
"team_alias:kill-switch",
|
||||
budget_windows_absent_cache_key("team", team_id),
|
||||
]
|
||||
assert (response.blocked, response.litellm_model_table, response.object_permission) == (blocked, None, None)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -14485,6 +14485,40 @@ async def test_late_requests_from_many_keys_with_windows_do_not_push_out_a_known
|
|||
assert no_window_key_reads() == reads_after_the_first_request
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_slow_window_row_read_does_not_hold_back_the_key_spend_counter():
|
||||
import litellm.proxy.proxy_server as ps
|
||||
from litellm.proxy.proxy_server import increment_spend_counters
|
||||
|
||||
prisma_client: Final = _prisma_with_rows()
|
||||
read_started: Final = asyncio.Event()
|
||||
finish_read: Final = asyncio.Event()
|
||||
|
||||
async def slow_row_read(where: Mapping[str, str]) -> _StoredRow:
|
||||
read_started.set()
|
||||
await finish_read.wait()
|
||||
return _StoredRow([_window("7d", 5.0)])
|
||||
|
||||
prisma_client.db.litellm_verificationtoken.find_unique = slow_row_read
|
||||
|
||||
with _window_spend_enqueue_env({}, prisma_client=prisma_client) as queue:
|
||||
ps.spend_counter_cache.set_cache(key="spend:key:hashed-token", value=0.0)
|
||||
ps.spend_counter_cache.set_cache(key="spend:key:hashed-token:window:7d", value=1.0)
|
||||
charge: Final = asyncio.create_task(
|
||||
increment_spend_counters(token="hashed-token", team_id=None, user_id=None, response_cost=0.25)
|
||||
)
|
||||
assert await asyncio.wait_for(read_started.wait(), timeout=5)
|
||||
key_spend_while_the_row_read_waits: Final = ps.spend_counter_cache.get_cache(key="spend:key:hashed-token")
|
||||
finish_read.set()
|
||||
await charge
|
||||
window_spend: Final = ps.spend_counter_cache.get_cache(key="spend:key:hashed-token:window:7d")
|
||||
charged: Final = await _drain(queue)
|
||||
|
||||
assert key_spend_while_the_row_read_waits == pytest.approx(0.25)
|
||||
assert window_spend == pytest.approx(1.25)
|
||||
assert _charged_windows(charged) == [("hashed-token", "7d", pytest.approx(0.25))]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_late_request_that_misses_the_updated_team_charges_windows_the_team_did_not_have_before():
|
||||
"""Another worker's /team/update only evicts this worker's cached team, so its next late cost reads the row."""
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue