refactor(redis): own enqueued reservation codec in typed store

This commit is contained in:
Devin AI 2026-10-30 17:52:18 +00:00
parent 0836f36cff
commit 13456df378
3 changed files with 207 additions and 64 deletions

View file

@ -13,7 +13,7 @@ import math
import time
import uuid
from collections.abc import Callable, Mapping
from dataclasses import dataclass, field
from dataclasses import dataclass
from datetime import timedelta
from typing import TYPE_CHECKING, Annotated, Final, Literal, TypeAlias
@ -23,14 +23,19 @@ from litellm._internal_context import with_service_target
from litellm._logging import verbose_proxy_logger
from litellm.constants import BATCH_ENQUEUED_TOKEN_LIMIT_METADATA_KEY, BATCH_ENQUEUED_TOKEN_TTL_SECONDS
from litellm.proxy._types import UserAPIKeyAuth
from litellm.redis.stores import enqueued_tokens
from litellm.redis.stores.enqueued_tokens import (
RESERVATION_ADAPTER,
AlreadyPopped,
EnqueuedTokensStore,
MalformedRecord,
OverLimit,
Popped,
Reserved,
TransportEnqueuedTokensStore,
counter_key,
decode_reservation,
encode_reservation,
record_key,
)
from litellm.redis.transport.contract import RedisFailure
@ -48,26 +53,10 @@ BATCH_ENQUEUED_REFUND_STATUSES: Final[frozenset[str]] = frozenset(
{"completed", "complete", "failed", "expired", "cancelled", "cancelling"}
)
ScopeKey: TypeAlias = Literal["api_key", "team"]
@dataclass(frozen=True, slots=True)
class BatchEnqueuedTokenScope:
key: ScopeKey
value: str
limit: int
ReservationBackend: TypeAlias = Literal["redis", "memory"]
@dataclass(frozen=True, slots=True)
class BatchEnqueuedTokenReservation:
tokens: int
scopes: tuple[BatchEnqueuedTokenScope, ...]
backend: ReservationBackend = "redis"
owner: str = ""
reserved_at_monotonic: float = field(default_factory=time.monotonic, compare=False)
ScopeKey: TypeAlias = enqueued_tokens.ScopeKey
ReservationBackend: TypeAlias = enqueued_tokens.ReservationBackend
BatchEnqueuedTokenScope: TypeAlias = enqueued_tokens.BatchEnqueuedTokenScope
BatchEnqueuedTokenReservation: TypeAlias = enqueued_tokens.BatchEnqueuedTokenReservation
@dataclass(frozen=True, slots=True)
@ -79,8 +68,8 @@ class BatchEnqueuedTokenOverLimit:
BatchEnqueuedTokenOutcome: TypeAlias = BatchEnqueuedTokenReservation | BatchEnqueuedTokenOverLimit
_LIMIT_ADAPTER: Final = TypeAdapter(Annotated[int, Field(gt=0)])
_RESERVATION_ADAPTER: Final = RESERVATION_ADAPTER
_STORED_COUNTER_ADAPTER: Final = TypeAdapter(int | None)
_RESERVATION_ADAPTER: Final = TypeAdapter(BatchEnqueuedTokenReservation)
_LOCAL_RECORD_ADAPTER: Final = TypeAdapter(object)
@ -309,16 +298,15 @@ class BatchEnqueuedTokenStore:
reservation: BatchEnqueuedTokenReservation,
litellm_parent_otel_span: "Span | None" = None,
) -> None:
serialized: Final = _RESERVATION_ADAPTER.dump_json(reservation).decode("utf-8")
elapsed: Final = self._monotonic() - reservation.reserved_at_monotonic
ttl: Final = max(1, BATCH_ENQUEUED_TOKEN_TTL_SECONDS - math.ceil(elapsed))
if self._store is not None:
saved: Final = await self._store.save_record(batch_id, serialized, ttl=timedelta(seconds=ttl))
saved: Final = await self._store.save_record(batch_id, reservation, ttl=timedelta(seconds=ttl))
if saved is None:
return
await self.internal_usage_cache.async_set_cache(
key=self._record_key(batch_id),
value=serialized,
value=encode_reservation(reservation),
ttl=ttl,
litellm_parent_otel_span=litellm_parent_otel_span,
local_only=True,
@ -337,22 +325,21 @@ class BatchEnqueuedTokenStore:
# left behind by a save that raised after landing must not refund again.
await self._pop_local_record(batch_id, litellm_parent_otel_span)
return None
raw: Final = (
popped.document
if isinstance(popped, Popped)
else await self._pop_local_record(batch_id, litellm_parent_otel_span)
)
if raw is None:
return None
try:
if isinstance(raw, (str, bytes)):
return _RESERVATION_ADAPTER.validate_json(raw)
return _RESERVATION_ADAPTER.validate_python(raw)
except ValidationError:
if isinstance(popped, Popped):
return popped.reservation
if isinstance(popped, MalformedRecord):
verbose_proxy_logger.warning("Discarding malformed enqueued-token reservation record for %s", batch_id)
return None
raw: Final = await self._pop_local_record(batch_id, litellm_parent_otel_span)
if raw is None:
return None
decoded: Final = decode_reservation(raw)
if isinstance(decoded, MalformedRecord):
verbose_proxy_logger.warning("Discarding malformed enqueued-token reservation record for %s", batch_id)
return None
return decoded
async def _pop_redis_record(self, batch_id: str) -> Popped | AlreadyPopped | None:
async def _pop_redis_record(self, batch_id: str) -> Popped | AlreadyPopped | MalformedRecord | None:
if self._store is None:
return None
popped: Final = await self._store.pop_record(

View file

@ -1,9 +1,34 @@
from dataclasses import dataclass
import time
from dataclasses import dataclass, field
from datetime import timedelta
from typing import Final, Protocol
from typing import Final, Literal, Protocol, TypeAlias
from pydantic import TypeAdapter, ValidationError
from litellm.redis.transport.contract import EvalReply, RedisFailure, RedisTransport
ScopeKey: TypeAlias = Literal["api_key", "team"]
ReservationBackend: TypeAlias = Literal["redis", "memory"]
@dataclass(frozen=True, slots=True)
class BatchEnqueuedTokenScope:
key: ScopeKey
value: str
limit: int
@dataclass(frozen=True, slots=True)
class BatchEnqueuedTokenReservation:
tokens: int
scopes: tuple[BatchEnqueuedTokenScope, ...]
backend: ReservationBackend = "redis"
owner: str = ""
reserved_at_monotonic: float = field(default_factory=time.monotonic, compare=False)
RESERVATION_ADAPTER: Final = TypeAdapter(BatchEnqueuedTokenReservation)
_RESERVE: Final = """
local amount = tonumber(ARGV[1])
local ttl = tonumber(ARGV[2])
@ -59,7 +84,12 @@ class OverLimit:
@dataclass(frozen=True, slots=True)
class Popped:
document: bytes
reservation: BatchEnqueuedTokenReservation
@dataclass(frozen=True, slots=True)
class MalformedRecord:
pass
@dataclass(frozen=True, slots=True)
@ -67,6 +97,19 @@ class AlreadyPopped:
pass
def encode_reservation(reservation: BatchEnqueuedTokenReservation) -> str:
return RESERVATION_ADAPTER.dump_json(reservation).decode("utf-8")
def decode_reservation(raw: object) -> BatchEnqueuedTokenReservation | MalformedRecord:
try:
if isinstance(raw, (str, bytes)):
return RESERVATION_ADAPTER.validate_json(raw)
return RESERVATION_ADAPTER.validate_python(raw)
except ValidationError:
return MalformedRecord()
class EnqueuedTokensStore(Protocol):
async def reserve(
self, scope_key: str, scope_value: str, tokens: int, *, limit: int, ttl: timedelta
@ -74,11 +117,13 @@ class EnqueuedTokensStore(Protocol):
async def refund(self, scope_key: str, scope_value: str, tokens: int) -> None | RedisFailure: ...
async def save_record(self, batch_id: str, document: str, *, ttl: timedelta) -> None | RedisFailure: ...
async def save_record(
self, batch_id: str, reservation: BatchEnqueuedTokenReservation, *, ttl: timedelta
) -> None | RedisFailure: ...
async def pop_record(
self, batch_id: str, *, tombstone_ttl: timedelta
) -> Popped | AlreadyPopped | None | RedisFailure: ...
) -> Popped | AlreadyPopped | MalformedRecord | None | RedisFailure: ...
@dataclass(frozen=True, slots=True)
@ -105,13 +150,17 @@ class TransportEnqueuedTokensStore:
reply: Final = await self.transport.eval(_REFUND, (counter_key(scope_key, scope_value),), (tokens,))
return reply if isinstance(reply, RedisFailure) else None
async def save_record(self, batch_id: str, document: str, *, ttl: timedelta) -> None | RedisFailure:
reply: Final = await self.transport.eval(_SAVE_RECORD, (record_key(batch_id),), (document, _seconds(ttl)))
async def save_record(
self, batch_id: str, reservation: BatchEnqueuedTokenReservation, *, ttl: timedelta
) -> None | RedisFailure:
reply: Final = await self.transport.eval(
_SAVE_RECORD, (record_key(batch_id),), (encode_reservation(reservation), _seconds(ttl))
)
return reply if isinstance(reply, RedisFailure) else None
async def pop_record(
self, batch_id: str, *, tombstone_ttl: timedelta
) -> Popped | AlreadyPopped | None | RedisFailure:
) -> Popped | AlreadyPopped | MalformedRecord | None | RedisFailure:
reply: Final = await self.transport.eval(_POP_RECORD, (record_key(batch_id),), (_seconds(tombstone_ttl),))
match reply:
case RedisFailure() | None:
@ -119,7 +168,8 @@ class TransportEnqueuedTokensStore:
case b"":
return AlreadyPopped()
case bytes():
return Popped(reply)
decoded: Final = decode_reservation(reply)
return decoded if isinstance(decoded, MalformedRecord) else Popped(decoded)
case _:
return _decode_failure("enqueued-token reservation record", reply)

View file

@ -171,8 +171,9 @@ def _redis_store(
fail_from_call: Mapping[str, int] = MappingProxyType({}),
raise_after_landing_keys: frozenset[str] = frozenset(),
monotonic: Callable[[], float] = time.monotonic,
namespace: str | None = None,
) -> tuple[BatchEnqueuedTokenStore, fakeredis.FakeAsyncRedis]:
cache = fake_redis_cache(server)
cache = fake_redis_cache(server, namespace=namespace)
raw = fakeredis.FakeAsyncRedis(server=server)
litellm.in_memory_llm_clients_cache.set_cache(
key=cache._get_async_client_cache_key(),
@ -223,19 +224,54 @@ async def test_a_counter_an_old_pod_wrote_counts_against_the_allowance():
assert await raw.get(_counter(scope)) == b"100"
@pytest.mark.asyncio
async def test_a_reservation_record_round_trips_with_old_pods():
store, raw = _redis_store(fakeredis.FakeServer())
scope = _scope(limit=100)
reservation = BatchEnqueuedTokenReservation(tokens=5, scopes=(scope,), backend="redis")
old_writer_document = _RESERVATION_ADAPTER.dump_json(reservation)
await raw.set(_record("batch_old"), old_writer_document, ex=60)
def _saved_reservation() -> BatchEnqueuedTokenReservation:
return BatchEnqueuedTokenReservation(
tokens=5,
scopes=(
BatchEnqueuedTokenScope(key="api_key", value='clé "one"', limit=100),
BatchEnqueuedTokenScope(key="team", value="組/二", limit=200),
),
backend="memory",
owner="worker\n雪",
reserved_at_monotonic=1000.25,
)
assert await store.pop_reservation("batch_old") == reservation
assert await raw.get(_record("batch_old")) == b""
_SAVED_DOCUMENT: Final = (
'{"tokens":5,"scopes":[{"key":"api_key","value":"clé \\"one\\"","limit":100},'
'{"key":"team","value":"組/二","limit":200}],"backend":"memory",'
'"owner":"worker\\n雪","reserved_at_monotonic":1000.25}'
)
@pytest.mark.asyncio
@pytest.mark.parametrize("namespace", (None, "test-namespace"))
async def test_a_reservation_record_round_trips_with_old_pods(
monkeypatch: pytest.MonkeyPatch, namespace: str | None
) -> None:
monkeypatch.setattr(time, "time", lambda: 1000.0)
store, raw = _redis_store(fakeredis.FakeServer(), monotonic=lambda: 1030.5, namespace=namespace)
reservation: Final = _saved_reservation()
prefix: Final = f"{namespace}:" if namespace else ""
old_key: Final = prefix + _record("batch_old")
new_key: Final = prefix + _record("batch_new")
await raw.set(old_key, _SAVED_DOCUMENT.encode("utf-8"), ex=60)
popped: Final = await store.pop_reservation("batch_old")
assert popped == reservation
assert popped.reserved_at_monotonic == reservation.reserved_at_monotonic
assert await raw.get(old_key) == b""
assert await raw.pttl(old_key) == BATCH_ENQUEUED_TOKEN_TTL_SECONDS * 1000
assert await store.pop_reservation("batch_old") is None
await store.save_reservation("batch_new", reservation)
assert _RESERVATION_ADAPTER.validate_json(await raw.get(_record("batch_new"))) == reservation
document: Final = await raw.get(new_key)
assert document == _SAVED_DOCUMENT.encode("utf-8")
old_reader: Final = _RESERVATION_ADAPTER.validate_json(document)
assert old_reader == reservation
assert old_reader.reserved_at_monotonic == reservation.reserved_at_monotonic
assert await raw.pttl(new_key) == (BATCH_ENQUEUED_TOKEN_TTL_SECONDS - 31) * 1000
assert set(await raw.keys("*")) == {old_key.encode(), new_key.encode()}
@pytest.mark.asyncio
@ -293,7 +329,9 @@ async def test_local_ghost_left_by_landed_save_never_refunds_twice():
reservation = await store.reserve(tokens=60, scopes=(scope,))
assert isinstance(reservation, BatchEnqueuedTokenReservation)
await store.save_reservation("batch_ghost", reservation)
assert await raw.get(_record("batch_ghost"))
saved: Final = _RESERVATION_ADAPTER.dump_json(reservation)
assert await raw.get(_record("batch_ghost")) == saved
assert store.internal_usage_cache.dual_cache.in_memory_cache.cache_dict[_record("batch_ghost")] == saved.decode()
first = await store.pop_reservation("batch_ghost")
assert first == reservation
@ -313,6 +351,70 @@ async def test_local_ghost_left_by_landed_save_never_refunds_twice():
assert await raw.get(_counter(scope)) == b"100"
@pytest.mark.asyncio
@pytest.mark.parametrize("document", (b"{", b"{}", b"null", b'"text"', b"\xff"))
async def test_consumed_malformed_record_does_not_refund_a_local_ghost(
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, document: bytes
) -> None:
monkeypatch.setattr(time, "time", lambda: 1000.0)
server: Final = fakeredis.FakeServer()
store, raw = _redis_store(server, namespace="test-namespace")
key: Final = _record("malformed")
wire_key: Final = f"test-namespace:{key}"
local: Final = store.internal_usage_cache.dual_cache.in_memory_cache
local.set_cache(key=key, value=_SAVED_DOCUMENT)
await raw.set(wire_key, document, ex=60)
assert await store.pop_reservation("malformed") is None
assert "Discarding malformed enqueued-token reservation record for malformed" in caplog.text
assert await raw.get(wire_key) == b""
assert await raw.pttl(wire_key) == BATCH_ENQUEUED_TOKEN_TTL_SECONDS * 1000
assert await store.pop_reservation("malformed") is None
assert local.get_cache(key) is None
server.connected = False
assert await store.pop_reservation("malformed") is None
@pytest.mark.asyncio
async def test_reservation_record_outage_keeps_exact_local_bytes_and_consumes_once() -> None:
server: Final = fakeredis.FakeServer()
store, raw = _redis_store(server, monotonic=lambda: 1000.25, namespace="test-namespace")
server.connected = False
await store.save_reservation("outage", _saved_reservation())
local: Final = store.internal_usage_cache.dual_cache.in_memory_cache
assert local.cache_dict[_record("outage")] == _SAVED_DOCUMENT
popped: Final = await store.pop_reservation("outage")
assert popped == _saved_reservation()
assert popped.reserved_at_monotonic == 1000.25
assert local.get_cache(_record("outage")) is None
assert await store.pop_reservation("outage") is None
server.connected = True
assert await raw.keys("*") == []
@pytest.mark.asyncio
@pytest.mark.parametrize("raw", (_SAVED_DOCUMENT, _SAVED_DOCUMENT.encode("utf-8")))
async def test_local_record_preserves_all_saved_fields(raw: str | bytes) -> None:
store: Final = _in_memory_store()
local: Final = store.internal_usage_cache.dual_cache.in_memory_cache
local.set_cache(key=_record("local"), value=raw)
popped: Final = await store.pop_reservation("local")
assert popped == _saved_reservation()
assert popped.reserved_at_monotonic == 1000.25
assert local.get_cache(_record("local")) is None
@pytest.mark.asyncio
@pytest.mark.parametrize("raw", ("{", b"{}", {"tokens": 5}, 0))
async def test_malformed_local_record_is_consumed_without_a_refund(raw: object) -> None:
store: Final = _in_memory_store()
local: Final = store.internal_usage_cache.dual_cache.in_memory_cache
local.set_cache(key=_record("local"), value=raw)
assert await store.pop_reservation("local") is None
assert local.get_cache(_record("local")) is None
@pytest.mark.asyncio
async def test_record_ttl_shrinks_by_elapsed_time_so_stale_records_never_outlive_their_counters():
scope = _scope(limit=100)
@ -360,16 +462,20 @@ async def test_failed_redis_refund_leaves_local_counters_untouched():
@pytest.mark.asyncio
async def test_pop_reservation_defaults_legacy_records_to_redis_backend():
store = _in_memory_store()
@pytest.mark.parametrize("redis_backed", (False, True))
async def test_pop_reservation_defaults_legacy_records_to_redis_backend(redis_backed: bool) -> None:
store, raw = _redis_store(fakeredis.FakeServer())
legacy = '{"tokens": 5, "scopes": [{"key": "api_key", "value": "k", "limit": 10}]}'
store.internal_usage_cache.dual_cache.in_memory_cache.set_cache(
key="batch_enqueued_token_reservation:batch_legacy", value=legacy
)
if redis_backed:
await raw.set(_record("batch_legacy"), legacy, ex=60)
else:
store.internal_usage_cache.dual_cache.in_memory_cache.set_cache(key=_record("batch_legacy"), value=legacy)
popped = await store.pop_reservation("batch_legacy")
assert popped == BatchEnqueuedTokenReservation(
tokens=5, scopes=(BatchEnqueuedTokenScope(key="api_key", value="k", limit=10),), backend="redis"
)
assert popped.owner == ""
assert isinstance(popped.reserved_at_monotonic, float)
def test_canonical_provider_batch_id_passes_raw_ids_through():