From a6fd07c4d6ea996af5eeb792f27db5cf06ebec3c Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 30 Oct 2026 16:15:43 +0000 Subject: [PATCH] refactor(redis-store): migrate batch enqueued-token accounting Co-authored-by: Yujong Lee --- litellm/proxy/hooks/batch_enqueued_tokens.py | 216 +++++----------- litellm/redis/stores/enqueued_tokens.py | 132 ++++++++++ .../proxy/hooks/test_batch_enqueued_tokens.py | 243 +++++++++--------- .../hooks/test_parallel_request_limiter_v3.py | 16 ++ 4 files changed, 332 insertions(+), 275 deletions(-) create mode 100644 litellm/redis/stores/enqueued_tokens.py diff --git a/litellm/proxy/hooks/batch_enqueued_tokens.py b/litellm/proxy/hooks/batch_enqueued_tokens.py index da595e101f1..ea6b5b8da99 100644 --- a/litellm/proxy/hooks/batch_enqueued_tokens.py +++ b/litellm/proxy/hooks/batch_enqueued_tokens.py @@ -9,21 +9,31 @@ the reservation is refunded when the batch reaches a terminal state """ import asyncio -import logging import math import time import uuid -from collections.abc import Awaitable, Callable, Mapping, Sequence +from collections.abc import Callable, Mapping from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Annotated, Final, Literal, Protocol, TypeAlias +from datetime import timedelta +from typing import TYPE_CHECKING, Annotated, Final, Literal, TypeAlias from pydantic import ConfigDict, Field, TypeAdapter, ValidationError from litellm._internal_context import with_service_target from litellm._logging import verbose_proxy_logger -from litellm.caching.redis_cache import log_redis_failure from litellm.constants import BATCH_ENQUEUED_TOKEN_LIMIT_METADATA_KEY, BATCH_ENQUEUED_TOKEN_TTL_SECONDS from litellm.proxy._types import UserAPIKeyAuth +from litellm.redis.stores.enqueued_tokens import ( + AlreadyPopped, + EnqueuedTokensStore, + OverLimit, + Popped, + Reserved, + TransportEnqueuedTokensStore, + counter_key, + record_key, +) +from litellm.redis.transport.contract import RedisFailure from litellm.types.llms.base import LiteLLMBaseModel if TYPE_CHECKING: @@ -40,40 +50,6 @@ BATCH_ENQUEUED_REFUND_STATUSES: Final[frozenset[str]] = frozenset( ScopeKey: TypeAlias = Literal["api_key", "team"] -RESERVE_ENQUEUED_TOKENS_SCRIPT: Final = """ -local amount = tonumber(ARGV[1]) -local ttl = tonumber(ARGV[2]) -local limit = tonumber(ARGV[3]) -local current = tonumber(redis.call('GET', KEYS[1]) or '0') -if current + amount > limit then - return {0, current} -end -local updated = redis.call('INCRBY', KEYS[1], amount) -redis.call('EXPIRE', KEYS[1], ttl) -return {1, updated} -""" - -REFUND_ENQUEUED_TOKENS_SCRIPT: Final = """ -local updated = redis.call('DECRBY', KEYS[1], tonumber(ARGV[1])) -if updated <= 0 then - redis.call('DEL', KEYS[1]) -end -return 1 -""" - -SAVE_RESERVATION_SCRIPT: Final = """ -redis.call('SET', KEYS[1], ARGV[1], 'EX', tonumber(ARGV[2])) -return 1 -""" - -POP_RESERVATION_SCRIPT: Final = """ -local value = redis.call('GET', KEYS[1]) -if value and value ~= '' then - redis.call('SET', KEYS[1], '', 'EX', tonumber(ARGV[1])) -end -return value -""" - @dataclass(frozen=True, slots=True) class BatchEnqueuedTokenScope: @@ -103,17 +79,11 @@ class BatchEnqueuedTokenOverLimit: BatchEnqueuedTokenOutcome: TypeAlias = BatchEnqueuedTokenReservation | BatchEnqueuedTokenOverLimit _LIMIT_ADAPTER: Final = TypeAdapter(Annotated[int, Field(gt=0)]) -_RESERVE_RESULT_ADAPTER: Final = TypeAdapter(tuple[int, int]) -_POPPED_VALUE_ADAPTER: Final = TypeAdapter(str | bytes | None) _STORED_COUNTER_ADAPTER: Final = TypeAdapter(int | None) _RESERVATION_ADAPTER: Final = TypeAdapter(BatchEnqueuedTokenReservation) _LOCAL_RECORD_ADAPTER: Final = TypeAdapter(object) -class _ScriptRunner(Protocol): - def __call__(self, keys: Sequence[str], args: Sequence[str | bytes | int | float]) -> Awaitable[object]: ... - - def _read_metadata_limit(metadata: Mapping[str, object] | None) -> int | None: if not metadata: return None @@ -202,27 +172,18 @@ class BatchEnqueuedTokenStore: self._monotonic: Final = monotonic self._lock = asyncio.Lock() self._owner_token = uuid.uuid4().hex - redis_cache = internal_usage_cache.dual_cache.redis_cache - self._reserve_script: _ScriptRunner | None = ( - redis_cache.async_register_script(RESERVE_ENQUEUED_TOKENS_SCRIPT) if redis_cache is not None else None - ) - self._refund_script: _ScriptRunner | None = ( - redis_cache.async_register_script(REFUND_ENQUEUED_TOKENS_SCRIPT) if redis_cache is not None else None - ) - self._save_script: _ScriptRunner | None = ( - redis_cache.async_register_script(SAVE_RESERVATION_SCRIPT) if redis_cache is not None else None - ) - self._pop_script: _ScriptRunner | None = ( - redis_cache.async_register_script(POP_RESERVATION_SCRIPT) if redis_cache is not None else None + redis_cache: Final = internal_usage_cache.dual_cache.redis_cache + self._store: Final[EnqueuedTokensStore | None] = ( + TransportEnqueuedTokensStore(redis_cache.transport()) if redis_cache is not None else None ) @staticmethod def _counter_key(scope: BatchEnqueuedTokenScope) -> str: - return f"batch_enqueued_tokens:{scope.key}:{scope.value}" + return counter_key(scope.key, scope.value) @staticmethod def _record_key(batch_id: str) -> str: - return f"batch_enqueued_token_reservation:{batch_id}" + return record_key(batch_id) @with_service_target("rate_limits") async def reserve( @@ -233,83 +194,58 @@ class BatchEnqueuedTokenStore: ) -> BatchEnqueuedTokenOutcome: if tokens <= 0 or not scopes: return BatchEnqueuedTokenReservation(tokens=max(tokens, 0), scopes=scopes) - reserve_script: Final = self._reserve_script - refund_script: Final = self._refund_script - if reserve_script is not None and refund_script is not None: - try: - return await self._reserve_via_redis(reserve_script, refund_script, tokens=tokens, scopes=scopes) - except Exception as e: # noqa: BLE001 # any Redis failure must fall back to the in-memory counters - log_redis_failure( - verbose_proxy_logger, - logging.WARNING, - "Redis enqueued-token reserve failed, falling back to in-memory", - e, - ) + if self._store is not None: + outcome: Final = await self._reserve_via_redis(self._store, tokens=tokens, scopes=scopes) + if not isinstance(outcome, RedisFailure): + return outcome return await self._reserve_in_memory(tokens=tokens, scopes=scopes, span=litellm_parent_otel_span) async def _reserve_via_redis( self, - reserve_script: _ScriptRunner, - refund_script: _ScriptRunner, + store: EnqueuedTokensStore, tokens: int, scopes: tuple[BatchEnqueuedTokenScope, ...], - ) -> BatchEnqueuedTokenOutcome: + ) -> BatchEnqueuedTokenOutcome | RedisFailure: started: Final = self._monotonic() + ttl: Final = timedelta(seconds=BATCH_ENQUEUED_TOKEN_TTL_SECONDS) for index, scope in enumerate(scopes): - result = await self._run_reserve_script( - reserve_script, - refund_script, - tokens=tokens, - scope=scope, - already_reserved=scopes[:index], - ) - if result[0] != 1: - await self._rollback_partial_reserve(refund_script, tokens=tokens, scopes=scopes[:index]) - return BatchEnqueuedTokenOverLimit(scope=scope, enqueued=result[1]) + reply = await store.reserve(scope.key, scope.value, tokens, limit=scope.limit, ttl=ttl) + match reply: + case Reserved(): + continue + case OverLimit(enqueued=enqueued): + await self._rollback_partial_reserve(store, tokens=tokens, scopes=scopes[:index]) + return BatchEnqueuedTokenOverLimit(scope=scope, enqueued=enqueued) + case RedisFailure(): + await self._rollback_partial_reserve(store, tokens=tokens, scopes=scopes[:index]) + return reply return BatchEnqueuedTokenReservation( tokens=tokens, scopes=scopes, backend="redis", reserved_at_monotonic=started ) - async def _run_reserve_script( - self, - reserve_script: _ScriptRunner, - refund_script: _ScriptRunner, - tokens: int, - scope: BatchEnqueuedTokenScope, - already_reserved: tuple[BatchEnqueuedTokenScope, ...], - ) -> tuple[int, int]: - try: - raw_result: Final = await reserve_script( - (self._counter_key(scope),), - (tokens, BATCH_ENQUEUED_TOKEN_TTL_SECONDS, scope.limit), - ) - return _RESERVE_RESULT_ADAPTER.validate_python(raw_result) - except Exception: - await self._rollback_partial_reserve(refund_script, tokens=tokens, scopes=already_reserved) - raise - async def _rollback_partial_reserve( self, - refund_script: _ScriptRunner, + store: EnqueuedTokensStore, tokens: int, scopes: tuple[BatchEnqueuedTokenScope, ...], ) -> None: - try: - await self._refund_via_redis(refund_script, tokens=tokens, scopes=scopes) - except Exception as e: # noqa: BLE001 # best-effort rollback: the leak is TTL-bounded and only tightens the allowance + refunded: Final = await self._refund_via_redis(store, tokens=tokens, scopes=scopes) + if isinstance(refunded, RedisFailure): verbose_proxy_logger.warning( - "Rollback of partially reserved enqueued tokens failed; leaked increments expire with the TTL: %s", - str(e), + "Rollback of partially reserved enqueued tokens failed; leaked increments expire with the TTL" ) async def _refund_via_redis( self, - refund_script: _ScriptRunner, + store: EnqueuedTokensStore, tokens: int, scopes: tuple[BatchEnqueuedTokenScope, ...], - ) -> None: + ) -> None | RedisFailure: for scope in scopes: - await refund_script((self._counter_key(scope),), (tokens,)) + refunded = await store.refund(scope.key, scope.value, tokens) + if refunded is not None: + return refunded + return None async def _reserve_in_memory( self, @@ -355,18 +291,16 @@ class BatchEnqueuedTokenStore: await self._set_local_counter(scope, remaining, litellm_parent_otel_span) async def _refund_redis_reservation(self, reservation: BatchEnqueuedTokenReservation) -> None: - refund_script: Final = self._refund_script - if refund_script is None: + if self._store is None: verbose_proxy_logger.warning( "No Redis client for a Redis-granted enqueued-token refund; leaked increments expire with the TTL" ) return - try: - await self._refund_via_redis(refund_script, tokens=reservation.tokens, scopes=reservation.scopes) - except Exception as e: # noqa: BLE001 # best-effort refund: the leak is TTL-bounded and only tightens the allowance - verbose_proxy_logger.warning( - "Redis enqueued-token refund failed; leaked increments expire with the TTL: %s", str(e) - ) + refunded: Final = await self._refund_via_redis( + self._store, tokens=reservation.tokens, scopes=reservation.scopes + ) + if isinstance(refunded, RedisFailure): + verbose_proxy_logger.warning("Redis enqueued-token refund failed; leaked increments expire with the TTL") @with_service_target("rate_limits") async def save_reservation( @@ -378,20 +312,9 @@ class BatchEnqueuedTokenStore: serialized: Final = _RESERVATION_ADAPTER.dump_json(reservation).decode("utf-8") elapsed: Final = self._monotonic() - reservation.reserved_at_monotonic ttl: Final = max(1, BATCH_ENQUEUED_TOKEN_TTL_SECONDS - math.ceil(elapsed)) - if self._save_script is not None: - try: - await self._save_script( - (self._record_key(batch_id),), - (serialized, ttl), - ) - except Exception as e: # noqa: BLE001 # any Redis failure must fall back to the in-memory record - log_redis_failure( - verbose_proxy_logger, - logging.WARNING, - "Redis enqueued-token reservation save failed, falling back to in-memory", - e, - ) - else: + if self._store is not None: + saved: Final = await self._store.save_record(batch_id, serialized, ttl=timedelta(seconds=ttl)) + if saved is None: return await self.internal_usage_cache.async_set_cache( key=self._record_key(batch_id), @@ -407,15 +330,17 @@ class BatchEnqueuedTokenStore: batch_id: str, litellm_parent_otel_span: "Span | None" = None, ) -> BatchEnqueuedTokenReservation | None: - redis_raw: Final = await self._pop_redis_record(batch_id) - if redis_raw is not None and not redis_raw: + popped: Final = await self._pop_redis_record(batch_id) + if isinstance(popped, AlreadyPopped): # The Redis pop tombstones popped records in place, so a hit on the empty # tombstone means the batch was already refunded elsewhere; a local copy # left behind by a save that raised after landing must not refund again. await self._pop_local_record(batch_id, litellm_parent_otel_span) return None raw: Final = ( - redis_raw if redis_raw is not None else await self._pop_local_record(batch_id, litellm_parent_otel_span) + popped.document + if isinstance(popped, Popped) + else await self._pop_local_record(batch_id, litellm_parent_otel_span) ) if raw is None: return None @@ -427,22 +352,13 @@ class BatchEnqueuedTokenStore: verbose_proxy_logger.warning("Discarding malformed enqueued-token reservation record for %s", batch_id) return None - async def _pop_redis_record(self, batch_id: str) -> str | bytes | None: - pop_script: Final = self._pop_script - if pop_script is None: - return None - try: - return _POPPED_VALUE_ADAPTER.validate_python( - await pop_script((self._record_key(batch_id),), (BATCH_ENQUEUED_TOKEN_TTL_SECONDS,)) - ) - except Exception as e: # noqa: BLE001 # any Redis failure must fall back to the in-memory record - log_redis_failure( - verbose_proxy_logger, - logging.WARNING, - "Redis enqueued-token reservation pop failed, falling back to in-memory", - e, - ) + async def _pop_redis_record(self, batch_id: str) -> Popped | AlreadyPopped | None: + if self._store is None: return None + popped: Final = await self._store.pop_record( + batch_id, tombstone_ttl=timedelta(seconds=BATCH_ENQUEUED_TOKEN_TTL_SECONDS) + ) + return None if isinstance(popped, RedisFailure) else popped async def _pop_local_record(self, batch_id: str, span: "Span | None") -> object: async with self._lock: diff --git a/litellm/redis/stores/enqueued_tokens.py b/litellm/redis/stores/enqueued_tokens.py new file mode 100644 index 00000000000..447808a953a --- /dev/null +++ b/litellm/redis/stores/enqueued_tokens.py @@ -0,0 +1,132 @@ +from dataclasses import dataclass +from datetime import timedelta +from typing import Final, Protocol + +from litellm.redis.transport.contract import EvalReply, RedisFailure, RedisTransport + +_RESERVE: Final = """ +local amount = tonumber(ARGV[1]) +local ttl = tonumber(ARGV[2]) +local limit = tonumber(ARGV[3]) +local current = tonumber(redis.call('GET', KEYS[1]) or '0') +if current + amount > limit then + return {0, current} +end +local updated = redis.call('INCRBY', KEYS[1], amount) +redis.call('EXPIRE', KEYS[1], ttl) +return {1, updated} +""" + +_REFUND: Final = """ +local updated = redis.call('DECRBY', KEYS[1], tonumber(ARGV[1])) +if updated <= 0 then + redis.call('DEL', KEYS[1]) +end +return 1 +""" + +_SAVE_RECORD: Final = """ +redis.call('SET', KEYS[1], ARGV[1], 'EX', tonumber(ARGV[2])) +return 1 +""" + +_POP_RECORD: Final = """ +local value = redis.call('GET', KEYS[1]) +if value and value ~= '' then + redis.call('SET', KEYS[1], '', 'EX', tonumber(ARGV[1])) +end +return value +""" + + +def counter_key(scope_key: str, scope_value: str) -> str: + return f"batch_enqueued_tokens:{scope_key}:{scope_value}" + + +def record_key(batch_id: str) -> str: + return f"batch_enqueued_token_reservation:{batch_id}" + + +@dataclass(frozen=True, slots=True) +class Reserved: + enqueued: int + + +@dataclass(frozen=True, slots=True) +class OverLimit: + enqueued: int + + +@dataclass(frozen=True, slots=True) +class Popped: + document: bytes + + +@dataclass(frozen=True, slots=True) +class AlreadyPopped: + pass + + +class EnqueuedTokensStore(Protocol): + async def reserve( + self, scope_key: str, scope_value: str, tokens: int, *, limit: int, ttl: timedelta + ) -> Reserved | OverLimit | RedisFailure: ... + + async def refund(self, scope_key: str, scope_value: str, tokens: int) -> None | RedisFailure: ... + + async def save_record(self, batch_id: str, document: str, *, ttl: timedelta) -> None | RedisFailure: ... + + async def pop_record( + self, batch_id: str, *, tombstone_ttl: timedelta + ) -> Popped | AlreadyPopped | None | RedisFailure: ... + + +@dataclass(frozen=True, slots=True) +class TransportEnqueuedTokensStore: + transport: RedisTransport + + async def reserve( + self, scope_key: str, scope_value: str, tokens: int, *, limit: int, ttl: timedelta + ) -> Reserved | OverLimit | RedisFailure: + reply: Final = await self.transport.eval( + _RESERVE, (counter_key(scope_key, scope_value),), (tokens, _seconds(ttl), limit) + ) + match reply: + case RedisFailure(): + return reply + case (1, int(enqueued)): + return Reserved(enqueued) + case (0, int(enqueued)): + return OverLimit(enqueued) + case _: + return _decode_failure("enqueued-token reserve", reply) + + async def refund(self, scope_key: str, scope_value: str, tokens: int) -> None | RedisFailure: + reply: Final = await self.transport.eval(_REFUND, (counter_key(scope_key, scope_value),), (tokens,)) + return reply if isinstance(reply, RedisFailure) else None + + async def save_record(self, batch_id: str, document: str, *, ttl: timedelta) -> None | RedisFailure: + reply: Final = await self.transport.eval(_SAVE_RECORD, (record_key(batch_id),), (document, _seconds(ttl))) + return reply if isinstance(reply, RedisFailure) else None + + async def pop_record( + self, batch_id: str, *, tombstone_ttl: timedelta + ) -> Popped | AlreadyPopped | None | RedisFailure: + reply: Final = await self.transport.eval(_POP_RECORD, (record_key(batch_id),), (_seconds(tombstone_ttl),)) + match reply: + case RedisFailure() | None: + return reply + case b"": + return AlreadyPopped() + case bytes(): + return Popped(reply) + case _: + return _decode_failure("enqueued-token reservation record", reply) + + +def _seconds(ttl: timedelta) -> int: + return ttl // timedelta(seconds=1) + + +def _decode_failure(what: str, reply: EvalReply) -> RedisFailure: + return RedisFailure(kind="decode", message=f"unexpected {what} reply: {reply!r}") diff --git a/tests/unit/proxy/hooks/test_batch_enqueued_tokens.py b/tests/unit/proxy/hooks/test_batch_enqueued_tokens.py index 40d49f5ab95..074d1679373 100644 --- a/tests/unit/proxy/hooks/test_batch_enqueued_tokens.py +++ b/tests/unit/proxy/hooks/test_batch_enqueued_tokens.py @@ -8,18 +8,25 @@ response-shape helpers the v3 limiter's post-call hooks rely on. import base64 import logging +import sys +import time import uuid -from collections.abc import Mapping, Sequence +from collections.abc import Callable, Mapping from types import MappingProxyType, SimpleNamespace from typing import Final +import fakeredis import pytest +from redis.exceptions import ConnectionError as RedisConnectionError +from redis.exceptions import NoScriptError +from redis.exceptions import TimeoutError as RedisTimeoutError +import litellm from litellm.caching.caching import DualCache -from litellm.caching.redis_cache import RedisCircuitBreakerOpenError from litellm.constants import BATCH_ENQUEUED_TOKEN_TTL_SECONDS from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.hooks.batch_enqueued_tokens import ( + _RESERVATION_ADAPTER, BatchEnqueuedTokenOverLimit, BatchEnqueuedTokenReservation, BatchEnqueuedTokenScope, @@ -29,6 +36,7 @@ from litellm.proxy.hooks.batch_enqueued_tokens import ( resolve_batch_enqueued_token_scopes, ) from litellm.proxy.utils import InternalUsageCache +from tests._support.fake_redis import fake_redis_cache def _in_memory_store() -> BatchEnqueuedTokenStore: @@ -127,113 +135,121 @@ async def test_zero_token_reserve_charges_nothing(): assert isinstance(full, BatchEnqueuedTokenReservation) -class _SingleKeyRedisFake: - """Emulates the Redis script path one single-key call at a time, recording every call.""" - +class _FaultyClient: def __init__( self, - fail_reserve_keys: frozenset[str] = frozenset(), - fail_refund_keys: frozenset[str] = frozenset(), - fail_save_keys: frozenset[str] = frozenset(), - raise_after_landing_save_keys: frozenset[str] = frozenset(), + inner: fakeredis.FakeAsyncRedis, + fail_from_call: Mapping[str, int], + raise_after_landing_keys: frozenset[str], ) -> None: - self.script_calls: tuple[tuple[str, tuple[str, ...]], ...] = () - self.save_ttls: tuple[int, ...] = () - self.counters: Mapping[str, int] = MappingProxyType({}) - self.records: Mapping[str, str] = MappingProxyType({}) - self.fail_reserve_keys = fail_reserve_keys - self.fail_refund_keys = fail_refund_keys - self.fail_save_keys = fail_save_keys - self.raise_after_landing_save_keys = raise_after_landing_save_keys + self.inner = inner + self.fail_from_call = fail_from_call + self.raise_after_landing_keys = raise_after_landing_keys + self.calls: Mapping[str, int] = MappingProxyType({}) - def async_register_script(self, script: str): - kind: Final = ( - "reserve" - if "INCRBY" in script - else "refund" if "DECRBY" in script else "pop" if "GET" in script else "save" - ) + async def evalsha(self, sha: str, numkeys: int, *keys_and_args: str | bytes | float) -> object: + if not (await self.inner.script_exists(sha))[0]: + raise NoScriptError("No matching script") + return await self._run(self.inner.evalsha, sha, numkeys, keys_and_args) - async def run(keys: Sequence[str], args: Sequence[str | bytes | int | float]) -> object: - self.script_calls = (*self.script_calls, (kind, tuple(keys))) - return self._run(kind, tuple(keys), tuple(args)) + async def eval(self, script: str, numkeys: int, *keys_and_args: str | bytes | float) -> object: + return await self._run(self.inner.eval, script, numkeys, keys_and_args) - return run + async def _run(self, run, script: str, numkeys: int, keys_and_args: tuple[str | bytes | int | float, ...]): + key: Final = str(keys_and_args[0]) + self.calls = MappingProxyType({**self.calls, key: self.calls.get(key, 0) + 1}) + if self.calls[key] >= self.fail_from_call.get(key, sys.maxsize): + raise RedisConnectionError(f"simulated redis failure for {key}") + reply: Final = await run(script, numkeys, *keys_and_args) + if key in self.raise_after_landing_keys: + raise RedisTimeoutError(f"simulated redis timeout after landing for {key}") + return reply - def _run(self, kind: str, keys: tuple[str, ...], args: tuple[str | bytes | int | float, ...]) -> object: - if kind == "reserve": - if keys[0] in self.fail_reserve_keys: - raise ConnectionError(f"simulated redis failure for {keys[0]}") - amount, limit = int(args[0]), int(args[2]) - current: Final = self.counters.get(keys[0], 0) - if current + amount > limit: - return (0, current) - self.counters = MappingProxyType({**self.counters, keys[0]: current + amount}) - return (1, current + amount) - if kind == "refund": - if keys[0] in self.fail_refund_keys: - raise ConnectionError(f"simulated redis failure for {keys[0]}") - remaining: Final = self.counters.get(keys[0], 0) - int(args[0]) - self.counters = MappingProxyType( - {key: value for key, value in self.counters.items() if key != keys[0]} - if remaining <= 0 - else {**self.counters, keys[0]: remaining} - ) - return 1 - if kind == "save": - if keys[0] in self.fail_save_keys: - raise ConnectionError(f"simulated redis failure for {keys[0]}") - self.records = MappingProxyType({**self.records, keys[0]: str(args[0])}) - self.save_ttls = (*self.save_ttls, int(args[1])) - if keys[0] in self.raise_after_landing_save_keys: - raise TimeoutError(f"simulated redis timeout after landing for {keys[0]}") - return 1 - if kind == "pop": - popped: Final = self.records.get(keys[0]) - if popped: - self.records = MappingProxyType({**self.records, keys[0]: ""}) - return popped - raise AssertionError(f"unexpected {kind} script call for keys {keys}") + +def _redis_store( + server: fakeredis.FakeServer, + fail_from_call: Mapping[str, int] = MappingProxyType({}), + raise_after_landing_keys: frozenset[str] = frozenset(), + monotonic: Callable[[], float] = time.monotonic, +) -> tuple[BatchEnqueuedTokenStore, fakeredis.FakeAsyncRedis]: + cache = fake_redis_cache(server) + raw = fakeredis.FakeAsyncRedis(server=server) + litellm.in_memory_llm_clients_cache.set_cache( + key=cache._get_async_client_cache_key(), + value=_FaultyClient(raw, fail_from_call=fail_from_call, raise_after_landing_keys=raise_after_landing_keys), + ) + store = BatchEnqueuedTokenStore( + internal_usage_cache=InternalUsageCache(DualCache(redis_cache=cache, default_in_memory_ttl=60)), + monotonic=monotonic, + ) + return store, raw + + +def _counter(scope: BatchEnqueuedTokenScope) -> str: + return f"batch_enqueued_tokens:{scope.key}:{scope.value}" + + +def _record(batch_id: str) -> str: + return f"batch_enqueued_token_reservation:{batch_id}" @pytest.mark.asyncio -async def test_redis_reserve_issues_single_key_calls_and_rolls_back_on_over_limit(): - fake = _SingleKeyRedisFake() - store = BatchEnqueuedTokenStore( - internal_usage_cache=InternalUsageCache(DualCache(redis_cache=fake, default_in_memory_ttl=60)) - ) +async def test_redis_reserve_rolls_back_on_over_limit_and_refund_clears_the_counters(): + store, raw = _redis_store(fakeredis.FakeServer()) key_scope = _scope(limit=100, key="api_key") team_scope = _scope(limit=50, key="team") over = await store.reserve(tokens=60, scopes=(key_scope, team_scope)) assert over == BatchEnqueuedTokenOverLimit(scope=team_scope, enqueued=0) - assert tuple(kind for kind, _ in fake.script_calls) == ("reserve", "reserve", "refund") - assert not fake.counters + assert await raw.keys("*") == [] fits = await store.reserve(tokens=50, scopes=(key_scope, team_scope)) assert isinstance(fits, BatchEnqueuedTokenReservation) + assert fits.backend == "redis" + assert await raw.get(_counter(key_scope)) == b"50" + assert 0 < await raw.pttl(_counter(team_scope)) <= BATCH_ENQUEUED_TOKEN_TTL_SECONDS * 1000 await store.refund(fits) - assert not fake.counters - assert all(len(keys) == 1 for _, keys in fake.script_calls) + assert await raw.keys("*") == [] + + +@pytest.mark.asyncio +async def test_a_counter_an_old_pod_wrote_counts_against_the_allowance(): + store, raw = _redis_store(fakeredis.FakeServer()) + scope = _scope(limit=100) + await raw.set(_counter(scope), 70, ex=BATCH_ENQUEUED_TOKEN_TTL_SECONDS) + + assert await store.reserve(tokens=40, scopes=(scope,)) == BatchEnqueuedTokenOverLimit(scope=scope, enqueued=70) + assert isinstance(await store.reserve(tokens=30, scopes=(scope,)), BatchEnqueuedTokenReservation) + assert await raw.get(_counter(scope)) == b"100" + + +@pytest.mark.asyncio +async def test_a_reservation_record_round_trips_with_old_pods(): + store, raw = _redis_store(fakeredis.FakeServer()) + scope = _scope(limit=100) + reservation = BatchEnqueuedTokenReservation(tokens=5, scopes=(scope,), backend="redis") + old_writer_document = _RESERVATION_ADAPTER.dump_json(reservation) + await raw.set(_record("batch_old"), old_writer_document, ex=60) + + assert await store.pop_reservation("batch_old") == reservation + assert await raw.get(_record("batch_old")) == b"" + + await store.save_reservation("batch_new", reservation) + assert _RESERVATION_ADAPTER.validate_json(await raw.get(_record("batch_new"))) == reservation @pytest.mark.asyncio async def test_partial_redis_reserve_failure_rolls_back_and_grants_in_memory(): key_scope = _scope(limit=100, key="api_key") team_scope = _scope(limit=50, key="team") - fake = _SingleKeyRedisFake(fail_reserve_keys=frozenset({f"batch_enqueued_tokens:team:{team_scope.value}"})) - store = BatchEnqueuedTokenStore( - internal_usage_cache=InternalUsageCache(DualCache(redis_cache=fake, default_in_memory_ttl=60)) - ) + store, raw = _redis_store(fakeredis.FakeServer(), fail_from_call=MappingProxyType({_counter(team_scope): 1})) outcome = await store.reserve(tokens=10, scopes=(key_scope, team_scope)) assert isinstance(outcome, BatchEnqueuedTokenReservation) assert outcome.backend == "memory" - assert tuple(kind for kind, _ in fake.script_calls) == ("reserve", "reserve", "refund") - assert not fake.counters + assert await raw.exists(_counter(key_scope)) == 0 await store.refund(outcome) - assert tuple(kind for kind, _ in fake.script_calls) == ("reserve", "reserve", "refund") - refilled = await store.reserve(tokens=50, scopes=(team_scope,)) assert isinstance(refilled, BatchEnqueuedTokenReservation) assert refilled.backend == "memory" @@ -243,84 +259,70 @@ async def test_partial_redis_reserve_failure_rolls_back_and_grants_in_memory(): async def test_over_limit_verdict_survives_a_failing_rollback(): key_scope = _scope(limit=100, key="api_key") team_scope = _scope(limit=5, key="team") - key_counter: Final = f"batch_enqueued_tokens:api_key:{key_scope.value}" - fake = _SingleKeyRedisFake(fail_refund_keys=frozenset({key_counter})) - store = BatchEnqueuedTokenStore( - internal_usage_cache=InternalUsageCache(DualCache(redis_cache=fake, default_in_memory_ttl=60)) - ) + store, raw = _redis_store(fakeredis.FakeServer(), fail_from_call=MappingProxyType({_counter(key_scope): 2})) outcome = await store.reserve(tokens=10, scopes=(key_scope, team_scope)) assert outcome == BatchEnqueuedTokenOverLimit(scope=team_scope, enqueued=0) - assert fake.counters == {key_counter: 10} + assert await raw.get(_counter(key_scope)) == b"10" @pytest.mark.asyncio -async def test_pop_falls_back_to_local_record_when_redis_pop_finds_nothing(): +async def test_pop_falls_back_to_local_record_when_the_redis_save_failed(): scope = _scope(limit=100) - record_key: Final = "batch_enqueued_token_reservation:batch_local_record" - fake = _SingleKeyRedisFake(fail_save_keys=frozenset({record_key})) - store = BatchEnqueuedTokenStore( - internal_usage_cache=InternalUsageCache(DualCache(redis_cache=fake, default_in_memory_ttl=60)) + store, raw = _redis_store( + fakeredis.FakeServer(), fail_from_call=MappingProxyType({_record("batch_local_record"): 1}) ) reservation = await store.reserve(tokens=60, scopes=(scope,)) assert isinstance(reservation, BatchEnqueuedTokenReservation) await store.save_reservation("batch_local_record", reservation) - assert not fake.records + assert await raw.exists(_record("batch_local_record")) == 0 popped = await store.pop_reservation("batch_local_record") assert popped == reservation await store.refund(popped) - assert not fake.counters + assert await raw.exists(_counter(scope)) == 0 assert await store.pop_reservation("batch_local_record") is None @pytest.mark.asyncio async def test_local_ghost_left_by_landed_save_never_refunds_twice(): scope = _scope(limit=100) - record_key: Final = "batch_enqueued_token_reservation:batch_ghost" - fake = _SingleKeyRedisFake(raise_after_landing_save_keys=frozenset({record_key})) - store = BatchEnqueuedTokenStore( - internal_usage_cache=InternalUsageCache(DualCache(redis_cache=fake, default_in_memory_ttl=60)) - ) + store, raw = _redis_store(fakeredis.FakeServer(), raise_after_landing_keys=frozenset({_record("batch_ghost")})) reservation = await store.reserve(tokens=60, scopes=(scope,)) assert isinstance(reservation, BatchEnqueuedTokenReservation) await store.save_reservation("batch_ghost", reservation) - assert fake.records[record_key] + assert await raw.get(_record("batch_ghost")) first = await store.pop_reservation("batch_ghost") assert first == reservation await store.refund(first) - assert not fake.counters - assert fake.records[record_key] == "" + assert await raw.exists(_counter(scope)) == 0 + assert await raw.get(_record("batch_ghost")) == b"" assert await store.pop_reservation("batch_ghost") is None assert ( await store.internal_usage_cache.async_get_cache( - key=record_key, litellm_parent_otel_span=None, local_only=True + key=_record("batch_ghost"), litellm_parent_otel_span=None, local_only=True ) is None ) assert await store.pop_reservation("batch_ghost") is None assert isinstance(await store.reserve(tokens=100, scopes=(scope,)), BatchEnqueuedTokenReservation) - assert fake.counters[f"batch_enqueued_tokens:{scope.key}:{scope.value}"] == 100 + assert await raw.get(_counter(scope)) == b"100" @pytest.mark.asyncio async def test_record_ttl_shrinks_by_elapsed_time_so_stale_records_never_outlive_their_counters(): scope = _scope(limit=100) - fake = _SingleKeyRedisFake() ticks = iter((1_000.0, 1_030.5)) - store = BatchEnqueuedTokenStore( - internal_usage_cache=InternalUsageCache(DualCache(redis_cache=fake, default_in_memory_ttl=60)), - monotonic=lambda: next(ticks), - ) + store, raw = _redis_store(fakeredis.FakeServer(), monotonic=lambda: next(ticks)) reservation = await store.reserve(tokens=60, scopes=(scope,)) assert isinstance(reservation, BatchEnqueuedTokenReservation) assert reservation.reserved_at_monotonic == 1_000.0 await store.save_reservation("batch_ttl_clamp", reservation) - assert fake.save_ttls == (BATCH_ENQUEUED_TOKEN_TTL_SECONDS - 31,) + assert await raw.ttl(_record("batch_ttl_clamp")) == BATCH_ENQUEUED_TOKEN_TTL_SECONDS - 31 assert await store.pop_reservation("batch_ttl_clamp") == reservation @@ -346,20 +348,15 @@ async def test_memory_refund_skips_reservations_granted_by_another_worker(): @pytest.mark.asyncio async def test_failed_redis_refund_leaves_local_counters_untouched(): scope = _scope(limit=100) - counter_key: Final = f"batch_enqueued_tokens:api_key:{scope.value}" - fake = _SingleKeyRedisFake(fail_refund_keys=frozenset({counter_key})) - store = BatchEnqueuedTokenStore( - internal_usage_cache=InternalUsageCache(DualCache(redis_cache=fake, default_in_memory_ttl=60)) - ) + store, raw = _redis_store(fakeredis.FakeServer(), fail_from_call=MappingProxyType({_counter(scope): 2})) reservation = await store.reserve(tokens=60, scopes=(scope,)) assert isinstance(reservation, BatchEnqueuedTokenReservation) assert reservation.backend == "redis" - store.internal_usage_cache.dual_cache.in_memory_cache.set_cache(key=counter_key, value=45) + store.internal_usage_cache.dual_cache.in_memory_cache.set_cache(key=_counter(scope), value=45) await store.refund(reservation) - assert store.internal_usage_cache.dual_cache.in_memory_cache.get_cache(key=counter_key) == 45 - assert fake.counters == {counter_key: 60} + assert store.internal_usage_cache.dual_cache.in_memory_cache.get_cache(key=_counter(scope)) == 45 @pytest.mark.asyncio @@ -401,22 +398,18 @@ def test_batch_response_view_accepts_batch_objects_only(): assert batch_response_view("batch_1") is None -class _OpenBreakerRedis: - def async_register_script(self, script: str): - async def refused(keys: Sequence[str], args: Sequence[str | bytes | int | float]) -> object: - raise RedisCircuitBreakerOpenError("Redis circuit breaker is open") - - return refused - - @pytest.mark.asyncio async def test_an_open_circuit_breaker_keeps_reservations_in_memory_without_a_warning(caplog): scope = _scope(limit=100) + cache = fake_redis_cache(fakeredis.FakeServer()) + for _ in range(cache._circuit_breaker.failure_threshold): + cache._circuit_breaker.record_failure() store = BatchEnqueuedTokenStore( - internal_usage_cache=InternalUsageCache(DualCache(redis_cache=_OpenBreakerRedis(), default_in_memory_ttl=60)) # pyright: ignore[reportArgumentType] # duck-typed Redis double + internal_usage_cache=InternalUsageCache(DualCache(redis_cache=cache, default_in_memory_ttl=60)) ) + caplog.clear() - with caplog.at_level(logging.DEBUG, logger="LiteLLM Proxy"): + with caplog.at_level(logging.DEBUG): reservation = await store.reserve(tokens=60, scopes=(scope,)) assert isinstance(reservation, BatchEnqueuedTokenReservation) assert reservation.backend == "memory" diff --git a/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py index 3aad2bda9bf..4e95f518b68 100644 --- a/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py @@ -15,6 +15,7 @@ from typing import Any, Dict, Final, List, Optional from unittest.mock import patch import httpx +import fakeredis import pytest from fastapi import HTTPException from pydantic import TypeAdapter @@ -7176,6 +7177,11 @@ async def test_post_call_success_hook_leaves_raw_provider_dict_untouched(): class _OpenBreakerRedis: + def transport(self): + from tests._support.fake_redis import fake_redis_transport + + return fake_redis_transport(fakeredis.FakeAsyncRedis(), failure_threshold=1) + def async_register_script(self, script: str): async def refused(keys, args): from litellm.caching.redis_cache import RedisCircuitBreakerOpenError @@ -7224,6 +7230,11 @@ async def test_an_open_circuit_breaker_reads_the_sliding_window_locally_without_ class _UnreachableRedis: + def transport(self): + from tests._support.fake_redis import fake_redis_transport + + return fake_redis_transport(fakeredis.FakeAsyncRedis(), failure_threshold=1) + def async_register_script(self, script: str): async def refused(keys, args): raise ConnectionError("Error 61 connecting to 127.0.0.1:6379. Connection refused.") @@ -7232,6 +7243,11 @@ class _UnreachableRedis: class _ScriptedRedis: + def transport(self): + from tests._support.fake_redis import fake_redis_transport + + return fake_redis_transport(fakeredis.FakeAsyncRedis(), failure_threshold=1) + def __init__( self, failing_script: str | None = None,