refactor(redis-store): migrate batch enqueued-token accounting

Co-authored-by: Yujong Lee <yujong@berri.ai>
This commit is contained in:
Devin AI 2026-10-30 16:15:43 +00:00
parent 015aa2e3d1
commit a6fd07c4d6
4 changed files with 332 additions and 275 deletions

View file

@ -9,21 +9,31 @@ the reservation is refunded when the batch reaches a terminal state
"""
import asyncio
import logging
import math
import time
import uuid
from collections.abc import Awaitable, Callable, Mapping, Sequence
from collections.abc import Callable, Mapping
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Annotated, Final, Literal, Protocol, TypeAlias
from datetime import timedelta
from typing import TYPE_CHECKING, Annotated, Final, Literal, TypeAlias
from pydantic import ConfigDict, Field, TypeAdapter, ValidationError
from litellm._internal_context import with_service_target
from litellm._logging import verbose_proxy_logger
from litellm.caching.redis_cache import log_redis_failure
from litellm.constants import BATCH_ENQUEUED_TOKEN_LIMIT_METADATA_KEY, BATCH_ENQUEUED_TOKEN_TTL_SECONDS
from litellm.proxy._types import UserAPIKeyAuth
from litellm.redis.stores.enqueued_tokens import (
AlreadyPopped,
EnqueuedTokensStore,
OverLimit,
Popped,
Reserved,
TransportEnqueuedTokensStore,
counter_key,
record_key,
)
from litellm.redis.transport.contract import RedisFailure
from litellm.types.llms.base import LiteLLMBaseModel
if TYPE_CHECKING:
@ -40,40 +50,6 @@ BATCH_ENQUEUED_REFUND_STATUSES: Final[frozenset[str]] = frozenset(
ScopeKey: TypeAlias = Literal["api_key", "team"]
RESERVE_ENQUEUED_TOKENS_SCRIPT: Final = """
local amount = tonumber(ARGV[1])
local ttl = tonumber(ARGV[2])
local limit = tonumber(ARGV[3])
local current = tonumber(redis.call('GET', KEYS[1]) or '0')
if current + amount > limit then
return {0, current}
end
local updated = redis.call('INCRBY', KEYS[1], amount)
redis.call('EXPIRE', KEYS[1], ttl)
return {1, updated}
"""
REFUND_ENQUEUED_TOKENS_SCRIPT: Final = """
local updated = redis.call('DECRBY', KEYS[1], tonumber(ARGV[1]))
if updated <= 0 then
redis.call('DEL', KEYS[1])
end
return 1
"""
SAVE_RESERVATION_SCRIPT: Final = """
redis.call('SET', KEYS[1], ARGV[1], 'EX', tonumber(ARGV[2]))
return 1
"""
POP_RESERVATION_SCRIPT: Final = """
local value = redis.call('GET', KEYS[1])
if value and value ~= '' then
redis.call('SET', KEYS[1], '', 'EX', tonumber(ARGV[1]))
end
return value
"""
@dataclass(frozen=True, slots=True)
class BatchEnqueuedTokenScope:
@ -103,17 +79,11 @@ class BatchEnqueuedTokenOverLimit:
BatchEnqueuedTokenOutcome: TypeAlias = BatchEnqueuedTokenReservation | BatchEnqueuedTokenOverLimit
_LIMIT_ADAPTER: Final = TypeAdapter(Annotated[int, Field(gt=0)])
_RESERVE_RESULT_ADAPTER: Final = TypeAdapter(tuple[int, int])
_POPPED_VALUE_ADAPTER: Final = TypeAdapter(str | bytes | None)
_STORED_COUNTER_ADAPTER: Final = TypeAdapter(int | None)
_RESERVATION_ADAPTER: Final = TypeAdapter(BatchEnqueuedTokenReservation)
_LOCAL_RECORD_ADAPTER: Final = TypeAdapter(object)
class _ScriptRunner(Protocol):
def __call__(self, keys: Sequence[str], args: Sequence[str | bytes | int | float]) -> Awaitable[object]: ...
def _read_metadata_limit(metadata: Mapping[str, object] | None) -> int | None:
if not metadata:
return None
@ -202,27 +172,18 @@ class BatchEnqueuedTokenStore:
self._monotonic: Final = monotonic
self._lock = asyncio.Lock()
self._owner_token = uuid.uuid4().hex
redis_cache = internal_usage_cache.dual_cache.redis_cache
self._reserve_script: _ScriptRunner | None = (
redis_cache.async_register_script(RESERVE_ENQUEUED_TOKENS_SCRIPT) if redis_cache is not None else None
)
self._refund_script: _ScriptRunner | None = (
redis_cache.async_register_script(REFUND_ENQUEUED_TOKENS_SCRIPT) if redis_cache is not None else None
)
self._save_script: _ScriptRunner | None = (
redis_cache.async_register_script(SAVE_RESERVATION_SCRIPT) if redis_cache is not None else None
)
self._pop_script: _ScriptRunner | None = (
redis_cache.async_register_script(POP_RESERVATION_SCRIPT) if redis_cache is not None else None
redis_cache: Final = internal_usage_cache.dual_cache.redis_cache
self._store: Final[EnqueuedTokensStore | None] = (
TransportEnqueuedTokensStore(redis_cache.transport()) if redis_cache is not None else None
)
@staticmethod
def _counter_key(scope: BatchEnqueuedTokenScope) -> str:
return f"batch_enqueued_tokens:{scope.key}:{scope.value}"
return counter_key(scope.key, scope.value)
@staticmethod
def _record_key(batch_id: str) -> str:
return f"batch_enqueued_token_reservation:{batch_id}"
return record_key(batch_id)
@with_service_target("rate_limits")
async def reserve(
@ -233,83 +194,58 @@ class BatchEnqueuedTokenStore:
) -> BatchEnqueuedTokenOutcome:
if tokens <= 0 or not scopes:
return BatchEnqueuedTokenReservation(tokens=max(tokens, 0), scopes=scopes)
reserve_script: Final = self._reserve_script
refund_script: Final = self._refund_script
if reserve_script is not None and refund_script is not None:
try:
return await self._reserve_via_redis(reserve_script, refund_script, tokens=tokens, scopes=scopes)
except Exception as e: # noqa: BLE001 # any Redis failure must fall back to the in-memory counters
log_redis_failure(
verbose_proxy_logger,
logging.WARNING,
"Redis enqueued-token reserve failed, falling back to in-memory",
e,
)
if self._store is not None:
outcome: Final = await self._reserve_via_redis(self._store, tokens=tokens, scopes=scopes)
if not isinstance(outcome, RedisFailure):
return outcome
return await self._reserve_in_memory(tokens=tokens, scopes=scopes, span=litellm_parent_otel_span)
async def _reserve_via_redis(
self,
reserve_script: _ScriptRunner,
refund_script: _ScriptRunner,
store: EnqueuedTokensStore,
tokens: int,
scopes: tuple[BatchEnqueuedTokenScope, ...],
) -> BatchEnqueuedTokenOutcome:
) -> BatchEnqueuedTokenOutcome | RedisFailure:
started: Final = self._monotonic()
ttl: Final = timedelta(seconds=BATCH_ENQUEUED_TOKEN_TTL_SECONDS)
for index, scope in enumerate(scopes):
result = await self._run_reserve_script(
reserve_script,
refund_script,
tokens=tokens,
scope=scope,
already_reserved=scopes[:index],
)
if result[0] != 1:
await self._rollback_partial_reserve(refund_script, tokens=tokens, scopes=scopes[:index])
return BatchEnqueuedTokenOverLimit(scope=scope, enqueued=result[1])
reply = await store.reserve(scope.key, scope.value, tokens, limit=scope.limit, ttl=ttl)
match reply:
case Reserved():
continue
case OverLimit(enqueued=enqueued):
await self._rollback_partial_reserve(store, tokens=tokens, scopes=scopes[:index])
return BatchEnqueuedTokenOverLimit(scope=scope, enqueued=enqueued)
case RedisFailure():
await self._rollback_partial_reserve(store, tokens=tokens, scopes=scopes[:index])
return reply
return BatchEnqueuedTokenReservation(
tokens=tokens, scopes=scopes, backend="redis", reserved_at_monotonic=started
)
async def _run_reserve_script(
self,
reserve_script: _ScriptRunner,
refund_script: _ScriptRunner,
tokens: int,
scope: BatchEnqueuedTokenScope,
already_reserved: tuple[BatchEnqueuedTokenScope, ...],
) -> tuple[int, int]:
try:
raw_result: Final = await reserve_script(
(self._counter_key(scope),),
(tokens, BATCH_ENQUEUED_TOKEN_TTL_SECONDS, scope.limit),
)
return _RESERVE_RESULT_ADAPTER.validate_python(raw_result)
except Exception:
await self._rollback_partial_reserve(refund_script, tokens=tokens, scopes=already_reserved)
raise
async def _rollback_partial_reserve(
self,
refund_script: _ScriptRunner,
store: EnqueuedTokensStore,
tokens: int,
scopes: tuple[BatchEnqueuedTokenScope, ...],
) -> None:
try:
await self._refund_via_redis(refund_script, tokens=tokens, scopes=scopes)
except Exception as e: # noqa: BLE001 # best-effort rollback: the leak is TTL-bounded and only tightens the allowance
refunded: Final = await self._refund_via_redis(store, tokens=tokens, scopes=scopes)
if isinstance(refunded, RedisFailure):
verbose_proxy_logger.warning(
"Rollback of partially reserved enqueued tokens failed; leaked increments expire with the TTL: %s",
str(e),
"Rollback of partially reserved enqueued tokens failed; leaked increments expire with the TTL"
)
async def _refund_via_redis(
self,
refund_script: _ScriptRunner,
store: EnqueuedTokensStore,
tokens: int,
scopes: tuple[BatchEnqueuedTokenScope, ...],
) -> None:
) -> None | RedisFailure:
for scope in scopes:
await refund_script((self._counter_key(scope),), (tokens,))
refunded = await store.refund(scope.key, scope.value, tokens)
if refunded is not None:
return refunded
return None
async def _reserve_in_memory(
self,
@ -355,18 +291,16 @@ class BatchEnqueuedTokenStore:
await self._set_local_counter(scope, remaining, litellm_parent_otel_span)
async def _refund_redis_reservation(self, reservation: BatchEnqueuedTokenReservation) -> None:
refund_script: Final = self._refund_script
if refund_script is None:
if self._store is None:
verbose_proxy_logger.warning(
"No Redis client for a Redis-granted enqueued-token refund; leaked increments expire with the TTL"
)
return
try:
await self._refund_via_redis(refund_script, tokens=reservation.tokens, scopes=reservation.scopes)
except Exception as e: # noqa: BLE001 # best-effort refund: the leak is TTL-bounded and only tightens the allowance
verbose_proxy_logger.warning(
"Redis enqueued-token refund failed; leaked increments expire with the TTL: %s", str(e)
)
refunded: Final = await self._refund_via_redis(
self._store, tokens=reservation.tokens, scopes=reservation.scopes
)
if isinstance(refunded, RedisFailure):
verbose_proxy_logger.warning("Redis enqueued-token refund failed; leaked increments expire with the TTL")
@with_service_target("rate_limits")
async def save_reservation(
@ -378,20 +312,9 @@ class BatchEnqueuedTokenStore:
serialized: Final = _RESERVATION_ADAPTER.dump_json(reservation).decode("utf-8")
elapsed: Final = self._monotonic() - reservation.reserved_at_monotonic
ttl: Final = max(1, BATCH_ENQUEUED_TOKEN_TTL_SECONDS - math.ceil(elapsed))
if self._save_script is not None:
try:
await self._save_script(
(self._record_key(batch_id),),
(serialized, ttl),
)
except Exception as e: # noqa: BLE001 # any Redis failure must fall back to the in-memory record
log_redis_failure(
verbose_proxy_logger,
logging.WARNING,
"Redis enqueued-token reservation save failed, falling back to in-memory",
e,
)
else:
if self._store is not None:
saved: Final = await self._store.save_record(batch_id, serialized, ttl=timedelta(seconds=ttl))
if saved is None:
return
await self.internal_usage_cache.async_set_cache(
key=self._record_key(batch_id),
@ -407,15 +330,17 @@ class BatchEnqueuedTokenStore:
batch_id: str,
litellm_parent_otel_span: "Span | None" = None,
) -> BatchEnqueuedTokenReservation | None:
redis_raw: Final = await self._pop_redis_record(batch_id)
if redis_raw is not None and not redis_raw:
popped: Final = await self._pop_redis_record(batch_id)
if isinstance(popped, AlreadyPopped):
# The Redis pop tombstones popped records in place, so a hit on the empty
# tombstone means the batch was already refunded elsewhere; a local copy
# left behind by a save that raised after landing must not refund again.
await self._pop_local_record(batch_id, litellm_parent_otel_span)
return None
raw: Final = (
redis_raw if redis_raw is not None else await self._pop_local_record(batch_id, litellm_parent_otel_span)
popped.document
if isinstance(popped, Popped)
else await self._pop_local_record(batch_id, litellm_parent_otel_span)
)
if raw is None:
return None
@ -427,22 +352,13 @@ class BatchEnqueuedTokenStore:
verbose_proxy_logger.warning("Discarding malformed enqueued-token reservation record for %s", batch_id)
return None
async def _pop_redis_record(self, batch_id: str) -> str | bytes | None:
pop_script: Final = self._pop_script
if pop_script is None:
return None
try:
return _POPPED_VALUE_ADAPTER.validate_python(
await pop_script((self._record_key(batch_id),), (BATCH_ENQUEUED_TOKEN_TTL_SECONDS,))
)
except Exception as e: # noqa: BLE001 # any Redis failure must fall back to the in-memory record
log_redis_failure(
verbose_proxy_logger,
logging.WARNING,
"Redis enqueued-token reservation pop failed, falling back to in-memory",
e,
)
async def _pop_redis_record(self, batch_id: str) -> Popped | AlreadyPopped | None:
if self._store is None:
return None
popped: Final = await self._store.pop_record(
batch_id, tombstone_ttl=timedelta(seconds=BATCH_ENQUEUED_TOKEN_TTL_SECONDS)
)
return None if isinstance(popped, RedisFailure) else popped
async def _pop_local_record(self, batch_id: str, span: "Span | None") -> object:
async with self._lock:

View file

@ -0,0 +1,132 @@
from dataclasses import dataclass
from datetime import timedelta
from typing import Final, Protocol
from litellm.redis.transport.contract import EvalReply, RedisFailure, RedisTransport
_RESERVE: Final = """
local amount = tonumber(ARGV[1])
local ttl = tonumber(ARGV[2])
local limit = tonumber(ARGV[3])
local current = tonumber(redis.call('GET', KEYS[1]) or '0')
if current + amount > limit then
return {0, current}
end
local updated = redis.call('INCRBY', KEYS[1], amount)
redis.call('EXPIRE', KEYS[1], ttl)
return {1, updated}
"""
_REFUND: Final = """
local updated = redis.call('DECRBY', KEYS[1], tonumber(ARGV[1]))
if updated <= 0 then
redis.call('DEL', KEYS[1])
end
return 1
"""
_SAVE_RECORD: Final = """
redis.call('SET', KEYS[1], ARGV[1], 'EX', tonumber(ARGV[2]))
return 1
"""
_POP_RECORD: Final = """
local value = redis.call('GET', KEYS[1])
if value and value ~= '' then
redis.call('SET', KEYS[1], '', 'EX', tonumber(ARGV[1]))
end
return value
"""
def counter_key(scope_key: str, scope_value: str) -> str:
return f"batch_enqueued_tokens:{scope_key}:{scope_value}"
def record_key(batch_id: str) -> str:
return f"batch_enqueued_token_reservation:{batch_id}"
@dataclass(frozen=True, slots=True)
class Reserved:
enqueued: int
@dataclass(frozen=True, slots=True)
class OverLimit:
enqueued: int
@dataclass(frozen=True, slots=True)
class Popped:
document: bytes
@dataclass(frozen=True, slots=True)
class AlreadyPopped:
pass
class EnqueuedTokensStore(Protocol):
async def reserve(
self, scope_key: str, scope_value: str, tokens: int, *, limit: int, ttl: timedelta
) -> Reserved | OverLimit | RedisFailure: ...
async def refund(self, scope_key: str, scope_value: str, tokens: int) -> None | RedisFailure: ...
async def save_record(self, batch_id: str, document: str, *, ttl: timedelta) -> None | RedisFailure: ...
async def pop_record(
self, batch_id: str, *, tombstone_ttl: timedelta
) -> Popped | AlreadyPopped | None | RedisFailure: ...
@dataclass(frozen=True, slots=True)
class TransportEnqueuedTokensStore:
transport: RedisTransport
async def reserve(
self, scope_key: str, scope_value: str, tokens: int, *, limit: int, ttl: timedelta
) -> Reserved | OverLimit | RedisFailure:
reply: Final = await self.transport.eval(
_RESERVE, (counter_key(scope_key, scope_value),), (tokens, _seconds(ttl), limit)
)
match reply:
case RedisFailure():
return reply
case (1, int(enqueued)):
return Reserved(enqueued)
case (0, int(enqueued)):
return OverLimit(enqueued)
case _:
return _decode_failure("enqueued-token reserve", reply)
async def refund(self, scope_key: str, scope_value: str, tokens: int) -> None | RedisFailure:
reply: Final = await self.transport.eval(_REFUND, (counter_key(scope_key, scope_value),), (tokens,))
return reply if isinstance(reply, RedisFailure) else None
async def save_record(self, batch_id: str, document: str, *, ttl: timedelta) -> None | RedisFailure:
reply: Final = await self.transport.eval(_SAVE_RECORD, (record_key(batch_id),), (document, _seconds(ttl)))
return reply if isinstance(reply, RedisFailure) else None
async def pop_record(
self, batch_id: str, *, tombstone_ttl: timedelta
) -> Popped | AlreadyPopped | None | RedisFailure:
reply: Final = await self.transport.eval(_POP_RECORD, (record_key(batch_id),), (_seconds(tombstone_ttl),))
match reply:
case RedisFailure() | None:
return reply
case b"":
return AlreadyPopped()
case bytes():
return Popped(reply)
case _:
return _decode_failure("enqueued-token reservation record", reply)
def _seconds(ttl: timedelta) -> int:
return ttl // timedelta(seconds=1)
def _decode_failure(what: str, reply: EvalReply) -> RedisFailure:
return RedisFailure(kind="decode", message=f"unexpected {what} reply: {reply!r}")

View file

@ -8,18 +8,25 @@ response-shape helpers the v3 limiter's post-call hooks rely on.
import base64
import logging
import sys
import time
import uuid
from collections.abc import Mapping, Sequence
from collections.abc import Callable, Mapping
from types import MappingProxyType, SimpleNamespace
from typing import Final
import fakeredis
import pytest
from redis.exceptions import ConnectionError as RedisConnectionError
from redis.exceptions import NoScriptError
from redis.exceptions import TimeoutError as RedisTimeoutError
import litellm
from litellm.caching.caching import DualCache
from litellm.caching.redis_cache import RedisCircuitBreakerOpenError
from litellm.constants import BATCH_ENQUEUED_TOKEN_TTL_SECONDS
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.hooks.batch_enqueued_tokens import (
_RESERVATION_ADAPTER,
BatchEnqueuedTokenOverLimit,
BatchEnqueuedTokenReservation,
BatchEnqueuedTokenScope,
@ -29,6 +36,7 @@ from litellm.proxy.hooks.batch_enqueued_tokens import (
resolve_batch_enqueued_token_scopes,
)
from litellm.proxy.utils import InternalUsageCache
from tests._support.fake_redis import fake_redis_cache
def _in_memory_store() -> BatchEnqueuedTokenStore:
@ -127,113 +135,121 @@ async def test_zero_token_reserve_charges_nothing():
assert isinstance(full, BatchEnqueuedTokenReservation)
class _SingleKeyRedisFake:
"""Emulates the Redis script path one single-key call at a time, recording every call."""
class _FaultyClient:
def __init__(
self,
fail_reserve_keys: frozenset[str] = frozenset(),
fail_refund_keys: frozenset[str] = frozenset(),
fail_save_keys: frozenset[str] = frozenset(),
raise_after_landing_save_keys: frozenset[str] = frozenset(),
inner: fakeredis.FakeAsyncRedis,
fail_from_call: Mapping[str, int],
raise_after_landing_keys: frozenset[str],
) -> None:
self.script_calls: tuple[tuple[str, tuple[str, ...]], ...] = ()
self.save_ttls: tuple[int, ...] = ()
self.counters: Mapping[str, int] = MappingProxyType({})
self.records: Mapping[str, str] = MappingProxyType({})
self.fail_reserve_keys = fail_reserve_keys
self.fail_refund_keys = fail_refund_keys
self.fail_save_keys = fail_save_keys
self.raise_after_landing_save_keys = raise_after_landing_save_keys
self.inner = inner
self.fail_from_call = fail_from_call
self.raise_after_landing_keys = raise_after_landing_keys
self.calls: Mapping[str, int] = MappingProxyType({})
def async_register_script(self, script: str):
kind: Final = (
"reserve"
if "INCRBY" in script
else "refund" if "DECRBY" in script else "pop" if "GET" in script else "save"
)
async def evalsha(self, sha: str, numkeys: int, *keys_and_args: str | bytes | float) -> object:
if not (await self.inner.script_exists(sha))[0]:
raise NoScriptError("No matching script")
return await self._run(self.inner.evalsha, sha, numkeys, keys_and_args)
async def run(keys: Sequence[str], args: Sequence[str | bytes | int | float]) -> object:
self.script_calls = (*self.script_calls, (kind, tuple(keys)))
return self._run(kind, tuple(keys), tuple(args))
async def eval(self, script: str, numkeys: int, *keys_and_args: str | bytes | float) -> object:
return await self._run(self.inner.eval, script, numkeys, keys_and_args)
return run
async def _run(self, run, script: str, numkeys: int, keys_and_args: tuple[str | bytes | int | float, ...]):
key: Final = str(keys_and_args[0])
self.calls = MappingProxyType({**self.calls, key: self.calls.get(key, 0) + 1})
if self.calls[key] >= self.fail_from_call.get(key, sys.maxsize):
raise RedisConnectionError(f"simulated redis failure for {key}")
reply: Final = await run(script, numkeys, *keys_and_args)
if key in self.raise_after_landing_keys:
raise RedisTimeoutError(f"simulated redis timeout after landing for {key}")
return reply
def _run(self, kind: str, keys: tuple[str, ...], args: tuple[str | bytes | int | float, ...]) -> object:
if kind == "reserve":
if keys[0] in self.fail_reserve_keys:
raise ConnectionError(f"simulated redis failure for {keys[0]}")
amount, limit = int(args[0]), int(args[2])
current: Final = self.counters.get(keys[0], 0)
if current + amount > limit:
return (0, current)
self.counters = MappingProxyType({**self.counters, keys[0]: current + amount})
return (1, current + amount)
if kind == "refund":
if keys[0] in self.fail_refund_keys:
raise ConnectionError(f"simulated redis failure for {keys[0]}")
remaining: Final = self.counters.get(keys[0], 0) - int(args[0])
self.counters = MappingProxyType(
{key: value for key, value in self.counters.items() if key != keys[0]}
if remaining <= 0
else {**self.counters, keys[0]: remaining}
)
return 1
if kind == "save":
if keys[0] in self.fail_save_keys:
raise ConnectionError(f"simulated redis failure for {keys[0]}")
self.records = MappingProxyType({**self.records, keys[0]: str(args[0])})
self.save_ttls = (*self.save_ttls, int(args[1]))
if keys[0] in self.raise_after_landing_save_keys:
raise TimeoutError(f"simulated redis timeout after landing for {keys[0]}")
return 1
if kind == "pop":
popped: Final = self.records.get(keys[0])
if popped:
self.records = MappingProxyType({**self.records, keys[0]: ""})
return popped
raise AssertionError(f"unexpected {kind} script call for keys {keys}")
def _redis_store(
server: fakeredis.FakeServer,
fail_from_call: Mapping[str, int] = MappingProxyType({}),
raise_after_landing_keys: frozenset[str] = frozenset(),
monotonic: Callable[[], float] = time.monotonic,
) -> tuple[BatchEnqueuedTokenStore, fakeredis.FakeAsyncRedis]:
cache = fake_redis_cache(server)
raw = fakeredis.FakeAsyncRedis(server=server)
litellm.in_memory_llm_clients_cache.set_cache(
key=cache._get_async_client_cache_key(),
value=_FaultyClient(raw, fail_from_call=fail_from_call, raise_after_landing_keys=raise_after_landing_keys),
)
store = BatchEnqueuedTokenStore(
internal_usage_cache=InternalUsageCache(DualCache(redis_cache=cache, default_in_memory_ttl=60)),
monotonic=monotonic,
)
return store, raw
def _counter(scope: BatchEnqueuedTokenScope) -> str:
return f"batch_enqueued_tokens:{scope.key}:{scope.value}"
def _record(batch_id: str) -> str:
return f"batch_enqueued_token_reservation:{batch_id}"
@pytest.mark.asyncio
async def test_redis_reserve_issues_single_key_calls_and_rolls_back_on_over_limit():
fake = _SingleKeyRedisFake()
store = BatchEnqueuedTokenStore(
internal_usage_cache=InternalUsageCache(DualCache(redis_cache=fake, default_in_memory_ttl=60))
)
async def test_redis_reserve_rolls_back_on_over_limit_and_refund_clears_the_counters():
store, raw = _redis_store(fakeredis.FakeServer())
key_scope = _scope(limit=100, key="api_key")
team_scope = _scope(limit=50, key="team")
over = await store.reserve(tokens=60, scopes=(key_scope, team_scope))
assert over == BatchEnqueuedTokenOverLimit(scope=team_scope, enqueued=0)
assert tuple(kind for kind, _ in fake.script_calls) == ("reserve", "reserve", "refund")
assert not fake.counters
assert await raw.keys("*") == []
fits = await store.reserve(tokens=50, scopes=(key_scope, team_scope))
assert isinstance(fits, BatchEnqueuedTokenReservation)
assert fits.backend == "redis"
assert await raw.get(_counter(key_scope)) == b"50"
assert 0 < await raw.pttl(_counter(team_scope)) <= BATCH_ENQUEUED_TOKEN_TTL_SECONDS * 1000
await store.refund(fits)
assert not fake.counters
assert all(len(keys) == 1 for _, keys in fake.script_calls)
assert await raw.keys("*") == []
@pytest.mark.asyncio
async def test_a_counter_an_old_pod_wrote_counts_against_the_allowance():
store, raw = _redis_store(fakeredis.FakeServer())
scope = _scope(limit=100)
await raw.set(_counter(scope), 70, ex=BATCH_ENQUEUED_TOKEN_TTL_SECONDS)
assert await store.reserve(tokens=40, scopes=(scope,)) == BatchEnqueuedTokenOverLimit(scope=scope, enqueued=70)
assert isinstance(await store.reserve(tokens=30, scopes=(scope,)), BatchEnqueuedTokenReservation)
assert await raw.get(_counter(scope)) == b"100"
@pytest.mark.asyncio
async def test_a_reservation_record_round_trips_with_old_pods():
store, raw = _redis_store(fakeredis.FakeServer())
scope = _scope(limit=100)
reservation = BatchEnqueuedTokenReservation(tokens=5, scopes=(scope,), backend="redis")
old_writer_document = _RESERVATION_ADAPTER.dump_json(reservation)
await raw.set(_record("batch_old"), old_writer_document, ex=60)
assert await store.pop_reservation("batch_old") == reservation
assert await raw.get(_record("batch_old")) == b""
await store.save_reservation("batch_new", reservation)
assert _RESERVATION_ADAPTER.validate_json(await raw.get(_record("batch_new"))) == reservation
@pytest.mark.asyncio
async def test_partial_redis_reserve_failure_rolls_back_and_grants_in_memory():
key_scope = _scope(limit=100, key="api_key")
team_scope = _scope(limit=50, key="team")
fake = _SingleKeyRedisFake(fail_reserve_keys=frozenset({f"batch_enqueued_tokens:team:{team_scope.value}"}))
store = BatchEnqueuedTokenStore(
internal_usage_cache=InternalUsageCache(DualCache(redis_cache=fake, default_in_memory_ttl=60))
)
store, raw = _redis_store(fakeredis.FakeServer(), fail_from_call=MappingProxyType({_counter(team_scope): 1}))
outcome = await store.reserve(tokens=10, scopes=(key_scope, team_scope))
assert isinstance(outcome, BatchEnqueuedTokenReservation)
assert outcome.backend == "memory"
assert tuple(kind for kind, _ in fake.script_calls) == ("reserve", "reserve", "refund")
assert not fake.counters
assert await raw.exists(_counter(key_scope)) == 0
await store.refund(outcome)
assert tuple(kind for kind, _ in fake.script_calls) == ("reserve", "reserve", "refund")
refilled = await store.reserve(tokens=50, scopes=(team_scope,))
assert isinstance(refilled, BatchEnqueuedTokenReservation)
assert refilled.backend == "memory"
@ -243,84 +259,70 @@ async def test_partial_redis_reserve_failure_rolls_back_and_grants_in_memory():
async def test_over_limit_verdict_survives_a_failing_rollback():
key_scope = _scope(limit=100, key="api_key")
team_scope = _scope(limit=5, key="team")
key_counter: Final = f"batch_enqueued_tokens:api_key:{key_scope.value}"
fake = _SingleKeyRedisFake(fail_refund_keys=frozenset({key_counter}))
store = BatchEnqueuedTokenStore(
internal_usage_cache=InternalUsageCache(DualCache(redis_cache=fake, default_in_memory_ttl=60))
)
store, raw = _redis_store(fakeredis.FakeServer(), fail_from_call=MappingProxyType({_counter(key_scope): 2}))
outcome = await store.reserve(tokens=10, scopes=(key_scope, team_scope))
assert outcome == BatchEnqueuedTokenOverLimit(scope=team_scope, enqueued=0)
assert fake.counters == {key_counter: 10}
assert await raw.get(_counter(key_scope)) == b"10"
@pytest.mark.asyncio
async def test_pop_falls_back_to_local_record_when_redis_pop_finds_nothing():
async def test_pop_falls_back_to_local_record_when_the_redis_save_failed():
scope = _scope(limit=100)
record_key: Final = "batch_enqueued_token_reservation:batch_local_record"
fake = _SingleKeyRedisFake(fail_save_keys=frozenset({record_key}))
store = BatchEnqueuedTokenStore(
internal_usage_cache=InternalUsageCache(DualCache(redis_cache=fake, default_in_memory_ttl=60))
store, raw = _redis_store(
fakeredis.FakeServer(), fail_from_call=MappingProxyType({_record("batch_local_record"): 1})
)
reservation = await store.reserve(tokens=60, scopes=(scope,))
assert isinstance(reservation, BatchEnqueuedTokenReservation)
await store.save_reservation("batch_local_record", reservation)
assert not fake.records
assert await raw.exists(_record("batch_local_record")) == 0
popped = await store.pop_reservation("batch_local_record")
assert popped == reservation
await store.refund(popped)
assert not fake.counters
assert await raw.exists(_counter(scope)) == 0
assert await store.pop_reservation("batch_local_record") is None
@pytest.mark.asyncio
async def test_local_ghost_left_by_landed_save_never_refunds_twice():
scope = _scope(limit=100)
record_key: Final = "batch_enqueued_token_reservation:batch_ghost"
fake = _SingleKeyRedisFake(raise_after_landing_save_keys=frozenset({record_key}))
store = BatchEnqueuedTokenStore(
internal_usage_cache=InternalUsageCache(DualCache(redis_cache=fake, default_in_memory_ttl=60))
)
store, raw = _redis_store(fakeredis.FakeServer(), raise_after_landing_keys=frozenset({_record("batch_ghost")}))
reservation = await store.reserve(tokens=60, scopes=(scope,))
assert isinstance(reservation, BatchEnqueuedTokenReservation)
await store.save_reservation("batch_ghost", reservation)
assert fake.records[record_key]
assert await raw.get(_record("batch_ghost"))
first = await store.pop_reservation("batch_ghost")
assert first == reservation
await store.refund(first)
assert not fake.counters
assert fake.records[record_key] == ""
assert await raw.exists(_counter(scope)) == 0
assert await raw.get(_record("batch_ghost")) == b""
assert await store.pop_reservation("batch_ghost") is None
assert (
await store.internal_usage_cache.async_get_cache(
key=record_key, litellm_parent_otel_span=None, local_only=True
key=_record("batch_ghost"), litellm_parent_otel_span=None, local_only=True
)
is None
)
assert await store.pop_reservation("batch_ghost") is None
assert isinstance(await store.reserve(tokens=100, scopes=(scope,)), BatchEnqueuedTokenReservation)
assert fake.counters[f"batch_enqueued_tokens:{scope.key}:{scope.value}"] == 100
assert await raw.get(_counter(scope)) == b"100"
@pytest.mark.asyncio
async def test_record_ttl_shrinks_by_elapsed_time_so_stale_records_never_outlive_their_counters():
scope = _scope(limit=100)
fake = _SingleKeyRedisFake()
ticks = iter((1_000.0, 1_030.5))
store = BatchEnqueuedTokenStore(
internal_usage_cache=InternalUsageCache(DualCache(redis_cache=fake, default_in_memory_ttl=60)),
monotonic=lambda: next(ticks),
)
store, raw = _redis_store(fakeredis.FakeServer(), monotonic=lambda: next(ticks))
reservation = await store.reserve(tokens=60, scopes=(scope,))
assert isinstance(reservation, BatchEnqueuedTokenReservation)
assert reservation.reserved_at_monotonic == 1_000.0
await store.save_reservation("batch_ttl_clamp", reservation)
assert fake.save_ttls == (BATCH_ENQUEUED_TOKEN_TTL_SECONDS - 31,)
assert await raw.ttl(_record("batch_ttl_clamp")) == BATCH_ENQUEUED_TOKEN_TTL_SECONDS - 31
assert await store.pop_reservation("batch_ttl_clamp") == reservation
@ -346,20 +348,15 @@ async def test_memory_refund_skips_reservations_granted_by_another_worker():
@pytest.mark.asyncio
async def test_failed_redis_refund_leaves_local_counters_untouched():
scope = _scope(limit=100)
counter_key: Final = f"batch_enqueued_tokens:api_key:{scope.value}"
fake = _SingleKeyRedisFake(fail_refund_keys=frozenset({counter_key}))
store = BatchEnqueuedTokenStore(
internal_usage_cache=InternalUsageCache(DualCache(redis_cache=fake, default_in_memory_ttl=60))
)
store, raw = _redis_store(fakeredis.FakeServer(), fail_from_call=MappingProxyType({_counter(scope): 2}))
reservation = await store.reserve(tokens=60, scopes=(scope,))
assert isinstance(reservation, BatchEnqueuedTokenReservation)
assert reservation.backend == "redis"
store.internal_usage_cache.dual_cache.in_memory_cache.set_cache(key=counter_key, value=45)
store.internal_usage_cache.dual_cache.in_memory_cache.set_cache(key=_counter(scope), value=45)
await store.refund(reservation)
assert store.internal_usage_cache.dual_cache.in_memory_cache.get_cache(key=counter_key) == 45
assert fake.counters == {counter_key: 60}
assert store.internal_usage_cache.dual_cache.in_memory_cache.get_cache(key=_counter(scope)) == 45
@pytest.mark.asyncio
@ -401,22 +398,18 @@ def test_batch_response_view_accepts_batch_objects_only():
assert batch_response_view("batch_1") is None
class _OpenBreakerRedis:
def async_register_script(self, script: str):
async def refused(keys: Sequence[str], args: Sequence[str | bytes | int | float]) -> object:
raise RedisCircuitBreakerOpenError("Redis circuit breaker is open")
return refused
@pytest.mark.asyncio
async def test_an_open_circuit_breaker_keeps_reservations_in_memory_without_a_warning(caplog):
scope = _scope(limit=100)
cache = fake_redis_cache(fakeredis.FakeServer())
for _ in range(cache._circuit_breaker.failure_threshold):
cache._circuit_breaker.record_failure()
store = BatchEnqueuedTokenStore(
internal_usage_cache=InternalUsageCache(DualCache(redis_cache=_OpenBreakerRedis(), default_in_memory_ttl=60)) # pyright: ignore[reportArgumentType] # duck-typed Redis double
internal_usage_cache=InternalUsageCache(DualCache(redis_cache=cache, default_in_memory_ttl=60))
)
caplog.clear()
with caplog.at_level(logging.DEBUG, logger="LiteLLM Proxy"):
with caplog.at_level(logging.DEBUG):
reservation = await store.reserve(tokens=60, scopes=(scope,))
assert isinstance(reservation, BatchEnqueuedTokenReservation)
assert reservation.backend == "memory"

View file

@ -15,6 +15,7 @@ from typing import Any, Dict, Final, List, Optional
from unittest.mock import patch
import httpx
import fakeredis
import pytest
from fastapi import HTTPException
from pydantic import TypeAdapter
@ -7176,6 +7177,11 @@ async def test_post_call_success_hook_leaves_raw_provider_dict_untouched():
class _OpenBreakerRedis:
def transport(self):
from tests._support.fake_redis import fake_redis_transport
return fake_redis_transport(fakeredis.FakeAsyncRedis(), failure_threshold=1)
def async_register_script(self, script: str):
async def refused(keys, args):
from litellm.caching.redis_cache import RedisCircuitBreakerOpenError
@ -7224,6 +7230,11 @@ async def test_an_open_circuit_breaker_reads_the_sliding_window_locally_without_
class _UnreachableRedis:
def transport(self):
from tests._support.fake_redis import fake_redis_transport
return fake_redis_transport(fakeredis.FakeAsyncRedis(), failure_threshold=1)
def async_register_script(self, script: str):
async def refused(keys, args):
raise ConnectionError("Error 61 connecting to 127.0.0.1:6379. Connection refused.")
@ -7232,6 +7243,11 @@ class _UnreachableRedis:
class _ScriptedRedis:
def transport(self):
from tests._support.fake_redis import fake_redis_transport
return fake_redis_transport(fakeredis.FakeAsyncRedis(), failure_threshold=1)
def __init__(
self,
failing_script: str | None = None,