From 13456df378ff7cad2fe8be56cb35c113c6038137 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 30 Oct 2026 17:52:18 +0000 Subject: [PATCH] refactor(redis): own enqueued reservation codec in typed store --- litellm/proxy/hooks/batch_enqueued_tokens.py | 63 ++++---- litellm/redis/stores/enqueued_tokens.py | 68 +++++++-- .../proxy/hooks/test_batch_enqueued_tokens.py | 140 +++++++++++++++--- 3 files changed, 207 insertions(+), 64 deletions(-) diff --git a/litellm/proxy/hooks/batch_enqueued_tokens.py b/litellm/proxy/hooks/batch_enqueued_tokens.py index ea6b5b8da99..c89e3458d21 100644 --- a/litellm/proxy/hooks/batch_enqueued_tokens.py +++ b/litellm/proxy/hooks/batch_enqueued_tokens.py @@ -13,7 +13,7 @@ import math import time import uuid from collections.abc import Callable, Mapping -from dataclasses import dataclass, field +from dataclasses import dataclass from datetime import timedelta from typing import TYPE_CHECKING, Annotated, Final, Literal, TypeAlias @@ -23,14 +23,19 @@ from litellm._internal_context import with_service_target from litellm._logging import verbose_proxy_logger from litellm.constants import BATCH_ENQUEUED_TOKEN_LIMIT_METADATA_KEY, BATCH_ENQUEUED_TOKEN_TTL_SECONDS from litellm.proxy._types import UserAPIKeyAuth +from litellm.redis.stores import enqueued_tokens from litellm.redis.stores.enqueued_tokens import ( + RESERVATION_ADAPTER, AlreadyPopped, EnqueuedTokensStore, + MalformedRecord, OverLimit, Popped, Reserved, TransportEnqueuedTokensStore, counter_key, + decode_reservation, + encode_reservation, record_key, ) from litellm.redis.transport.contract import RedisFailure @@ -48,26 +53,10 @@ BATCH_ENQUEUED_REFUND_STATUSES: Final[frozenset[str]] = frozenset( {"completed", "complete", "failed", "expired", "cancelled", "cancelling"} ) -ScopeKey: TypeAlias = Literal["api_key", "team"] - - -@dataclass(frozen=True, slots=True) -class BatchEnqueuedTokenScope: - key: ScopeKey - value: str - limit: int - - -ReservationBackend: TypeAlias = Literal["redis", "memory"] - - -@dataclass(frozen=True, slots=True) -class BatchEnqueuedTokenReservation: - tokens: int - scopes: tuple[BatchEnqueuedTokenScope, ...] - backend: ReservationBackend = "redis" - owner: str = "" - reserved_at_monotonic: float = field(default_factory=time.monotonic, compare=False) +ScopeKey: TypeAlias = enqueued_tokens.ScopeKey +ReservationBackend: TypeAlias = enqueued_tokens.ReservationBackend +BatchEnqueuedTokenScope: TypeAlias = enqueued_tokens.BatchEnqueuedTokenScope +BatchEnqueuedTokenReservation: TypeAlias = enqueued_tokens.BatchEnqueuedTokenReservation @dataclass(frozen=True, slots=True) @@ -79,8 +68,8 @@ class BatchEnqueuedTokenOverLimit: BatchEnqueuedTokenOutcome: TypeAlias = BatchEnqueuedTokenReservation | BatchEnqueuedTokenOverLimit _LIMIT_ADAPTER: Final = TypeAdapter(Annotated[int, Field(gt=0)]) +_RESERVATION_ADAPTER: Final = RESERVATION_ADAPTER _STORED_COUNTER_ADAPTER: Final = TypeAdapter(int | None) -_RESERVATION_ADAPTER: Final = TypeAdapter(BatchEnqueuedTokenReservation) _LOCAL_RECORD_ADAPTER: Final = TypeAdapter(object) @@ -309,16 +298,15 @@ class BatchEnqueuedTokenStore: reservation: BatchEnqueuedTokenReservation, litellm_parent_otel_span: "Span | None" = None, ) -> None: - serialized: Final = _RESERVATION_ADAPTER.dump_json(reservation).decode("utf-8") elapsed: Final = self._monotonic() - reservation.reserved_at_monotonic ttl: Final = max(1, BATCH_ENQUEUED_TOKEN_TTL_SECONDS - math.ceil(elapsed)) if self._store is not None: - saved: Final = await self._store.save_record(batch_id, serialized, ttl=timedelta(seconds=ttl)) + saved: Final = await self._store.save_record(batch_id, reservation, ttl=timedelta(seconds=ttl)) if saved is None: return await self.internal_usage_cache.async_set_cache( key=self._record_key(batch_id), - value=serialized, + value=encode_reservation(reservation), ttl=ttl, litellm_parent_otel_span=litellm_parent_otel_span, local_only=True, @@ -337,22 +325,21 @@ class BatchEnqueuedTokenStore: # left behind by a save that raised after landing must not refund again. await self._pop_local_record(batch_id, litellm_parent_otel_span) return None - raw: Final = ( - popped.document - if isinstance(popped, Popped) - else await self._pop_local_record(batch_id, litellm_parent_otel_span) - ) - if raw is None: - return None - try: - if isinstance(raw, (str, bytes)): - return _RESERVATION_ADAPTER.validate_json(raw) - return _RESERVATION_ADAPTER.validate_python(raw) - except ValidationError: + if isinstance(popped, Popped): + return popped.reservation + if isinstance(popped, MalformedRecord): verbose_proxy_logger.warning("Discarding malformed enqueued-token reservation record for %s", batch_id) return None + raw: Final = await self._pop_local_record(batch_id, litellm_parent_otel_span) + if raw is None: + return None + decoded: Final = decode_reservation(raw) + if isinstance(decoded, MalformedRecord): + verbose_proxy_logger.warning("Discarding malformed enqueued-token reservation record for %s", batch_id) + return None + return decoded - async def _pop_redis_record(self, batch_id: str) -> Popped | AlreadyPopped | None: + async def _pop_redis_record(self, batch_id: str) -> Popped | AlreadyPopped | MalformedRecord | None: if self._store is None: return None popped: Final = await self._store.pop_record( diff --git a/litellm/redis/stores/enqueued_tokens.py b/litellm/redis/stores/enqueued_tokens.py index 447808a953a..94be9dcd65e 100644 --- a/litellm/redis/stores/enqueued_tokens.py +++ b/litellm/redis/stores/enqueued_tokens.py @@ -1,9 +1,34 @@ -from dataclasses import dataclass +import time +from dataclasses import dataclass, field from datetime import timedelta -from typing import Final, Protocol +from typing import Final, Literal, Protocol, TypeAlias + +from pydantic import TypeAdapter, ValidationError from litellm.redis.transport.contract import EvalReply, RedisFailure, RedisTransport +ScopeKey: TypeAlias = Literal["api_key", "team"] +ReservationBackend: TypeAlias = Literal["redis", "memory"] + + +@dataclass(frozen=True, slots=True) +class BatchEnqueuedTokenScope: + key: ScopeKey + value: str + limit: int + + +@dataclass(frozen=True, slots=True) +class BatchEnqueuedTokenReservation: + tokens: int + scopes: tuple[BatchEnqueuedTokenScope, ...] + backend: ReservationBackend = "redis" + owner: str = "" + reserved_at_monotonic: float = field(default_factory=time.monotonic, compare=False) + + +RESERVATION_ADAPTER: Final = TypeAdapter(BatchEnqueuedTokenReservation) + _RESERVE: Final = """ local amount = tonumber(ARGV[1]) local ttl = tonumber(ARGV[2]) @@ -59,7 +84,12 @@ class OverLimit: @dataclass(frozen=True, slots=True) class Popped: - document: bytes + reservation: BatchEnqueuedTokenReservation + + +@dataclass(frozen=True, slots=True) +class MalformedRecord: + pass @dataclass(frozen=True, slots=True) @@ -67,6 +97,19 @@ class AlreadyPopped: pass +def encode_reservation(reservation: BatchEnqueuedTokenReservation) -> str: + return RESERVATION_ADAPTER.dump_json(reservation).decode("utf-8") + + +def decode_reservation(raw: object) -> BatchEnqueuedTokenReservation | MalformedRecord: + try: + if isinstance(raw, (str, bytes)): + return RESERVATION_ADAPTER.validate_json(raw) + return RESERVATION_ADAPTER.validate_python(raw) + except ValidationError: + return MalformedRecord() + + class EnqueuedTokensStore(Protocol): async def reserve( self, scope_key: str, scope_value: str, tokens: int, *, limit: int, ttl: timedelta @@ -74,11 +117,13 @@ class EnqueuedTokensStore(Protocol): async def refund(self, scope_key: str, scope_value: str, tokens: int) -> None | RedisFailure: ... - async def save_record(self, batch_id: str, document: str, *, ttl: timedelta) -> None | RedisFailure: ... + async def save_record( + self, batch_id: str, reservation: BatchEnqueuedTokenReservation, *, ttl: timedelta + ) -> None | RedisFailure: ... async def pop_record( self, batch_id: str, *, tombstone_ttl: timedelta - ) -> Popped | AlreadyPopped | None | RedisFailure: ... + ) -> Popped | AlreadyPopped | MalformedRecord | None | RedisFailure: ... @dataclass(frozen=True, slots=True) @@ -105,13 +150,17 @@ class TransportEnqueuedTokensStore: reply: Final = await self.transport.eval(_REFUND, (counter_key(scope_key, scope_value),), (tokens,)) return reply if isinstance(reply, RedisFailure) else None - async def save_record(self, batch_id: str, document: str, *, ttl: timedelta) -> None | RedisFailure: - reply: Final = await self.transport.eval(_SAVE_RECORD, (record_key(batch_id),), (document, _seconds(ttl))) + async def save_record( + self, batch_id: str, reservation: BatchEnqueuedTokenReservation, *, ttl: timedelta + ) -> None | RedisFailure: + reply: Final = await self.transport.eval( + _SAVE_RECORD, (record_key(batch_id),), (encode_reservation(reservation), _seconds(ttl)) + ) return reply if isinstance(reply, RedisFailure) else None async def pop_record( self, batch_id: str, *, tombstone_ttl: timedelta - ) -> Popped | AlreadyPopped | None | RedisFailure: + ) -> Popped | AlreadyPopped | MalformedRecord | None | RedisFailure: reply: Final = await self.transport.eval(_POP_RECORD, (record_key(batch_id),), (_seconds(tombstone_ttl),)) match reply: case RedisFailure() | None: @@ -119,7 +168,8 @@ class TransportEnqueuedTokensStore: case b"": return AlreadyPopped() case bytes(): - return Popped(reply) + decoded: Final = decode_reservation(reply) + return decoded if isinstance(decoded, MalformedRecord) else Popped(decoded) case _: return _decode_failure("enqueued-token reservation record", reply) diff --git a/tests/unit/proxy/hooks/test_batch_enqueued_tokens.py b/tests/unit/proxy/hooks/test_batch_enqueued_tokens.py index 074d1679373..345039fa54f 100644 --- a/tests/unit/proxy/hooks/test_batch_enqueued_tokens.py +++ b/tests/unit/proxy/hooks/test_batch_enqueued_tokens.py @@ -171,8 +171,9 @@ def _redis_store( fail_from_call: Mapping[str, int] = MappingProxyType({}), raise_after_landing_keys: frozenset[str] = frozenset(), monotonic: Callable[[], float] = time.monotonic, + namespace: str | None = None, ) -> tuple[BatchEnqueuedTokenStore, fakeredis.FakeAsyncRedis]: - cache = fake_redis_cache(server) + cache = fake_redis_cache(server, namespace=namespace) raw = fakeredis.FakeAsyncRedis(server=server) litellm.in_memory_llm_clients_cache.set_cache( key=cache._get_async_client_cache_key(), @@ -223,19 +224,54 @@ async def test_a_counter_an_old_pod_wrote_counts_against_the_allowance(): assert await raw.get(_counter(scope)) == b"100" -@pytest.mark.asyncio -async def test_a_reservation_record_round_trips_with_old_pods(): - store, raw = _redis_store(fakeredis.FakeServer()) - scope = _scope(limit=100) - reservation = BatchEnqueuedTokenReservation(tokens=5, scopes=(scope,), backend="redis") - old_writer_document = _RESERVATION_ADAPTER.dump_json(reservation) - await raw.set(_record("batch_old"), old_writer_document, ex=60) +def _saved_reservation() -> BatchEnqueuedTokenReservation: + return BatchEnqueuedTokenReservation( + tokens=5, + scopes=( + BatchEnqueuedTokenScope(key="api_key", value='clé "one"', limit=100), + BatchEnqueuedTokenScope(key="team", value="組/二", limit=200), + ), + backend="memory", + owner="worker\n雪", + reserved_at_monotonic=1000.25, + ) - assert await store.pop_reservation("batch_old") == reservation - assert await raw.get(_record("batch_old")) == b"" + +_SAVED_DOCUMENT: Final = ( + '{"tokens":5,"scopes":[{"key":"api_key","value":"clé \\"one\\"","limit":100},' + '{"key":"team","value":"組/二","limit":200}],"backend":"memory",' + '"owner":"worker\\n雪","reserved_at_monotonic":1000.25}' +) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("namespace", (None, "test-namespace")) +async def test_a_reservation_record_round_trips_with_old_pods( + monkeypatch: pytest.MonkeyPatch, namespace: str | None +) -> None: + monkeypatch.setattr(time, "time", lambda: 1000.0) + store, raw = _redis_store(fakeredis.FakeServer(), monotonic=lambda: 1030.5, namespace=namespace) + reservation: Final = _saved_reservation() + prefix: Final = f"{namespace}:" if namespace else "" + old_key: Final = prefix + _record("batch_old") + new_key: Final = prefix + _record("batch_new") + await raw.set(old_key, _SAVED_DOCUMENT.encode("utf-8"), ex=60) + + popped: Final = await store.pop_reservation("batch_old") + assert popped == reservation + assert popped.reserved_at_monotonic == reservation.reserved_at_monotonic + assert await raw.get(old_key) == b"" + assert await raw.pttl(old_key) == BATCH_ENQUEUED_TOKEN_TTL_SECONDS * 1000 + assert await store.pop_reservation("batch_old") is None await store.save_reservation("batch_new", reservation) - assert _RESERVATION_ADAPTER.validate_json(await raw.get(_record("batch_new"))) == reservation + document: Final = await raw.get(new_key) + assert document == _SAVED_DOCUMENT.encode("utf-8") + old_reader: Final = _RESERVATION_ADAPTER.validate_json(document) + assert old_reader == reservation + assert old_reader.reserved_at_monotonic == reservation.reserved_at_monotonic + assert await raw.pttl(new_key) == (BATCH_ENQUEUED_TOKEN_TTL_SECONDS - 31) * 1000 + assert set(await raw.keys("*")) == {old_key.encode(), new_key.encode()} @pytest.mark.asyncio @@ -293,7 +329,9 @@ async def test_local_ghost_left_by_landed_save_never_refunds_twice(): reservation = await store.reserve(tokens=60, scopes=(scope,)) assert isinstance(reservation, BatchEnqueuedTokenReservation) await store.save_reservation("batch_ghost", reservation) - assert await raw.get(_record("batch_ghost")) + saved: Final = _RESERVATION_ADAPTER.dump_json(reservation) + assert await raw.get(_record("batch_ghost")) == saved + assert store.internal_usage_cache.dual_cache.in_memory_cache.cache_dict[_record("batch_ghost")] == saved.decode() first = await store.pop_reservation("batch_ghost") assert first == reservation @@ -313,6 +351,70 @@ async def test_local_ghost_left_by_landed_save_never_refunds_twice(): assert await raw.get(_counter(scope)) == b"100" +@pytest.mark.asyncio +@pytest.mark.parametrize("document", (b"{", b"{}", b"null", b'"text"', b"\xff")) +async def test_consumed_malformed_record_does_not_refund_a_local_ghost( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, document: bytes +) -> None: + monkeypatch.setattr(time, "time", lambda: 1000.0) + server: Final = fakeredis.FakeServer() + store, raw = _redis_store(server, namespace="test-namespace") + key: Final = _record("malformed") + wire_key: Final = f"test-namespace:{key}" + local: Final = store.internal_usage_cache.dual_cache.in_memory_cache + local.set_cache(key=key, value=_SAVED_DOCUMENT) + await raw.set(wire_key, document, ex=60) + + assert await store.pop_reservation("malformed") is None + assert "Discarding malformed enqueued-token reservation record for malformed" in caplog.text + assert await raw.get(wire_key) == b"" + assert await raw.pttl(wire_key) == BATCH_ENQUEUED_TOKEN_TTL_SECONDS * 1000 + assert await store.pop_reservation("malformed") is None + assert local.get_cache(key) is None + server.connected = False + assert await store.pop_reservation("malformed") is None + + +@pytest.mark.asyncio +async def test_reservation_record_outage_keeps_exact_local_bytes_and_consumes_once() -> None: + server: Final = fakeredis.FakeServer() + store, raw = _redis_store(server, monotonic=lambda: 1000.25, namespace="test-namespace") + server.connected = False + await store.save_reservation("outage", _saved_reservation()) + local: Final = store.internal_usage_cache.dual_cache.in_memory_cache + assert local.cache_dict[_record("outage")] == _SAVED_DOCUMENT + + popped: Final = await store.pop_reservation("outage") + assert popped == _saved_reservation() + assert popped.reserved_at_monotonic == 1000.25 + assert local.get_cache(_record("outage")) is None + assert await store.pop_reservation("outage") is None + server.connected = True + assert await raw.keys("*") == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("raw", (_SAVED_DOCUMENT, _SAVED_DOCUMENT.encode("utf-8"))) +async def test_local_record_preserves_all_saved_fields(raw: str | bytes) -> None: + store: Final = _in_memory_store() + local: Final = store.internal_usage_cache.dual_cache.in_memory_cache + local.set_cache(key=_record("local"), value=raw) + popped: Final = await store.pop_reservation("local") + assert popped == _saved_reservation() + assert popped.reserved_at_monotonic == 1000.25 + assert local.get_cache(_record("local")) is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize("raw", ("{", b"{}", {"tokens": 5}, 0)) +async def test_malformed_local_record_is_consumed_without_a_refund(raw: object) -> None: + store: Final = _in_memory_store() + local: Final = store.internal_usage_cache.dual_cache.in_memory_cache + local.set_cache(key=_record("local"), value=raw) + assert await store.pop_reservation("local") is None + assert local.get_cache(_record("local")) is None + + @pytest.mark.asyncio async def test_record_ttl_shrinks_by_elapsed_time_so_stale_records_never_outlive_their_counters(): scope = _scope(limit=100) @@ -360,16 +462,20 @@ async def test_failed_redis_refund_leaves_local_counters_untouched(): @pytest.mark.asyncio -async def test_pop_reservation_defaults_legacy_records_to_redis_backend(): - store = _in_memory_store() +@pytest.mark.parametrize("redis_backed", (False, True)) +async def test_pop_reservation_defaults_legacy_records_to_redis_backend(redis_backed: bool) -> None: + store, raw = _redis_store(fakeredis.FakeServer()) legacy = '{"tokens": 5, "scopes": [{"key": "api_key", "value": "k", "limit": 10}]}' - store.internal_usage_cache.dual_cache.in_memory_cache.set_cache( - key="batch_enqueued_token_reservation:batch_legacy", value=legacy - ) + if redis_backed: + await raw.set(_record("batch_legacy"), legacy, ex=60) + else: + store.internal_usage_cache.dual_cache.in_memory_cache.set_cache(key=_record("batch_legacy"), value=legacy) popped = await store.pop_reservation("batch_legacy") assert popped == BatchEnqueuedTokenReservation( tokens=5, scopes=(BatchEnqueuedTokenScope(key="api_key", value="k", limit=10),), backend="redis" ) + assert popped.owner == "" + assert isinstance(popped.reserved_at_monotonic, float) def test_canonical_provider_batch_id_passes_raw_ids_through():