mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
refactor(redis): own enqueued reservation codec in typed store
This commit is contained in:
parent
0836f36cff
commit
13456df378
3 changed files with 207 additions and 64 deletions
|
|
@ -13,7 +13,7 @@ import math
|
|||
import time
|
||||
import uuid
|
||||
from collections.abc import Callable, Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from dataclasses import dataclass
|
||||
from datetime import timedelta
|
||||
from typing import TYPE_CHECKING, Annotated, Final, Literal, TypeAlias
|
||||
|
||||
|
|
@ -23,14 +23,19 @@ from litellm._internal_context import with_service_target
|
|||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.constants import BATCH_ENQUEUED_TOKEN_LIMIT_METADATA_KEY, BATCH_ENQUEUED_TOKEN_TTL_SECONDS
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.redis.stores import enqueued_tokens
|
||||
from litellm.redis.stores.enqueued_tokens import (
|
||||
RESERVATION_ADAPTER,
|
||||
AlreadyPopped,
|
||||
EnqueuedTokensStore,
|
||||
MalformedRecord,
|
||||
OverLimit,
|
||||
Popped,
|
||||
Reserved,
|
||||
TransportEnqueuedTokensStore,
|
||||
counter_key,
|
||||
decode_reservation,
|
||||
encode_reservation,
|
||||
record_key,
|
||||
)
|
||||
from litellm.redis.transport.contract import RedisFailure
|
||||
|
|
@ -48,26 +53,10 @@ BATCH_ENQUEUED_REFUND_STATUSES: Final[frozenset[str]] = frozenset(
|
|||
{"completed", "complete", "failed", "expired", "cancelled", "cancelling"}
|
||||
)
|
||||
|
||||
ScopeKey: TypeAlias = Literal["api_key", "team"]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class BatchEnqueuedTokenScope:
|
||||
key: ScopeKey
|
||||
value: str
|
||||
limit: int
|
||||
|
||||
|
||||
ReservationBackend: TypeAlias = Literal["redis", "memory"]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class BatchEnqueuedTokenReservation:
|
||||
tokens: int
|
||||
scopes: tuple[BatchEnqueuedTokenScope, ...]
|
||||
backend: ReservationBackend = "redis"
|
||||
owner: str = ""
|
||||
reserved_at_monotonic: float = field(default_factory=time.monotonic, compare=False)
|
||||
ScopeKey: TypeAlias = enqueued_tokens.ScopeKey
|
||||
ReservationBackend: TypeAlias = enqueued_tokens.ReservationBackend
|
||||
BatchEnqueuedTokenScope: TypeAlias = enqueued_tokens.BatchEnqueuedTokenScope
|
||||
BatchEnqueuedTokenReservation: TypeAlias = enqueued_tokens.BatchEnqueuedTokenReservation
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
@ -79,8 +68,8 @@ class BatchEnqueuedTokenOverLimit:
|
|||
BatchEnqueuedTokenOutcome: TypeAlias = BatchEnqueuedTokenReservation | BatchEnqueuedTokenOverLimit
|
||||
|
||||
_LIMIT_ADAPTER: Final = TypeAdapter(Annotated[int, Field(gt=0)])
|
||||
_RESERVATION_ADAPTER: Final = RESERVATION_ADAPTER
|
||||
_STORED_COUNTER_ADAPTER: Final = TypeAdapter(int | None)
|
||||
_RESERVATION_ADAPTER: Final = TypeAdapter(BatchEnqueuedTokenReservation)
|
||||
_LOCAL_RECORD_ADAPTER: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
|
|
@ -309,16 +298,15 @@ class BatchEnqueuedTokenStore:
|
|||
reservation: BatchEnqueuedTokenReservation,
|
||||
litellm_parent_otel_span: "Span | None" = None,
|
||||
) -> None:
|
||||
serialized: Final = _RESERVATION_ADAPTER.dump_json(reservation).decode("utf-8")
|
||||
elapsed: Final = self._monotonic() - reservation.reserved_at_monotonic
|
||||
ttl: Final = max(1, BATCH_ENQUEUED_TOKEN_TTL_SECONDS - math.ceil(elapsed))
|
||||
if self._store is not None:
|
||||
saved: Final = await self._store.save_record(batch_id, serialized, ttl=timedelta(seconds=ttl))
|
||||
saved: Final = await self._store.save_record(batch_id, reservation, ttl=timedelta(seconds=ttl))
|
||||
if saved is None:
|
||||
return
|
||||
await self.internal_usage_cache.async_set_cache(
|
||||
key=self._record_key(batch_id),
|
||||
value=serialized,
|
||||
value=encode_reservation(reservation),
|
||||
ttl=ttl,
|
||||
litellm_parent_otel_span=litellm_parent_otel_span,
|
||||
local_only=True,
|
||||
|
|
@ -337,22 +325,21 @@ class BatchEnqueuedTokenStore:
|
|||
# left behind by a save that raised after landing must not refund again.
|
||||
await self._pop_local_record(batch_id, litellm_parent_otel_span)
|
||||
return None
|
||||
raw: Final = (
|
||||
popped.document
|
||||
if isinstance(popped, Popped)
|
||||
else await self._pop_local_record(batch_id, litellm_parent_otel_span)
|
||||
)
|
||||
if raw is None:
|
||||
return None
|
||||
try:
|
||||
if isinstance(raw, (str, bytes)):
|
||||
return _RESERVATION_ADAPTER.validate_json(raw)
|
||||
return _RESERVATION_ADAPTER.validate_python(raw)
|
||||
except ValidationError:
|
||||
if isinstance(popped, Popped):
|
||||
return popped.reservation
|
||||
if isinstance(popped, MalformedRecord):
|
||||
verbose_proxy_logger.warning("Discarding malformed enqueued-token reservation record for %s", batch_id)
|
||||
return None
|
||||
raw: Final = await self._pop_local_record(batch_id, litellm_parent_otel_span)
|
||||
if raw is None:
|
||||
return None
|
||||
decoded: Final = decode_reservation(raw)
|
||||
if isinstance(decoded, MalformedRecord):
|
||||
verbose_proxy_logger.warning("Discarding malformed enqueued-token reservation record for %s", batch_id)
|
||||
return None
|
||||
return decoded
|
||||
|
||||
async def _pop_redis_record(self, batch_id: str) -> Popped | AlreadyPopped | None:
|
||||
async def _pop_redis_record(self, batch_id: str) -> Popped | AlreadyPopped | MalformedRecord | None:
|
||||
if self._store is None:
|
||||
return None
|
||||
popped: Final = await self._store.pop_record(
|
||||
|
|
|
|||
|
|
@ -1,9 +1,34 @@
|
|||
from dataclasses import dataclass
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import timedelta
|
||||
from typing import Final, Protocol
|
||||
from typing import Final, Literal, Protocol, TypeAlias
|
||||
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
from litellm.redis.transport.contract import EvalReply, RedisFailure, RedisTransport
|
||||
|
||||
ScopeKey: TypeAlias = Literal["api_key", "team"]
|
||||
ReservationBackend: TypeAlias = Literal["redis", "memory"]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class BatchEnqueuedTokenScope:
|
||||
key: ScopeKey
|
||||
value: str
|
||||
limit: int
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class BatchEnqueuedTokenReservation:
|
||||
tokens: int
|
||||
scopes: tuple[BatchEnqueuedTokenScope, ...]
|
||||
backend: ReservationBackend = "redis"
|
||||
owner: str = ""
|
||||
reserved_at_monotonic: float = field(default_factory=time.monotonic, compare=False)
|
||||
|
||||
|
||||
RESERVATION_ADAPTER: Final = TypeAdapter(BatchEnqueuedTokenReservation)
|
||||
|
||||
_RESERVE: Final = """
|
||||
local amount = tonumber(ARGV[1])
|
||||
local ttl = tonumber(ARGV[2])
|
||||
|
|
@ -59,7 +84,12 @@ class OverLimit:
|
|||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Popped:
|
||||
document: bytes
|
||||
reservation: BatchEnqueuedTokenReservation
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class MalformedRecord:
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
@ -67,6 +97,19 @@ class AlreadyPopped:
|
|||
pass
|
||||
|
||||
|
||||
def encode_reservation(reservation: BatchEnqueuedTokenReservation) -> str:
|
||||
return RESERVATION_ADAPTER.dump_json(reservation).decode("utf-8")
|
||||
|
||||
|
||||
def decode_reservation(raw: object) -> BatchEnqueuedTokenReservation | MalformedRecord:
|
||||
try:
|
||||
if isinstance(raw, (str, bytes)):
|
||||
return RESERVATION_ADAPTER.validate_json(raw)
|
||||
return RESERVATION_ADAPTER.validate_python(raw)
|
||||
except ValidationError:
|
||||
return MalformedRecord()
|
||||
|
||||
|
||||
class EnqueuedTokensStore(Protocol):
|
||||
async def reserve(
|
||||
self, scope_key: str, scope_value: str, tokens: int, *, limit: int, ttl: timedelta
|
||||
|
|
@ -74,11 +117,13 @@ class EnqueuedTokensStore(Protocol):
|
|||
|
||||
async def refund(self, scope_key: str, scope_value: str, tokens: int) -> None | RedisFailure: ...
|
||||
|
||||
async def save_record(self, batch_id: str, document: str, *, ttl: timedelta) -> None | RedisFailure: ...
|
||||
async def save_record(
|
||||
self, batch_id: str, reservation: BatchEnqueuedTokenReservation, *, ttl: timedelta
|
||||
) -> None | RedisFailure: ...
|
||||
|
||||
async def pop_record(
|
||||
self, batch_id: str, *, tombstone_ttl: timedelta
|
||||
) -> Popped | AlreadyPopped | None | RedisFailure: ...
|
||||
) -> Popped | AlreadyPopped | MalformedRecord | None | RedisFailure: ...
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
@ -105,13 +150,17 @@ class TransportEnqueuedTokensStore:
|
|||
reply: Final = await self.transport.eval(_REFUND, (counter_key(scope_key, scope_value),), (tokens,))
|
||||
return reply if isinstance(reply, RedisFailure) else None
|
||||
|
||||
async def save_record(self, batch_id: str, document: str, *, ttl: timedelta) -> None | RedisFailure:
|
||||
reply: Final = await self.transport.eval(_SAVE_RECORD, (record_key(batch_id),), (document, _seconds(ttl)))
|
||||
async def save_record(
|
||||
self, batch_id: str, reservation: BatchEnqueuedTokenReservation, *, ttl: timedelta
|
||||
) -> None | RedisFailure:
|
||||
reply: Final = await self.transport.eval(
|
||||
_SAVE_RECORD, (record_key(batch_id),), (encode_reservation(reservation), _seconds(ttl))
|
||||
)
|
||||
return reply if isinstance(reply, RedisFailure) else None
|
||||
|
||||
async def pop_record(
|
||||
self, batch_id: str, *, tombstone_ttl: timedelta
|
||||
) -> Popped | AlreadyPopped | None | RedisFailure:
|
||||
) -> Popped | AlreadyPopped | MalformedRecord | None | RedisFailure:
|
||||
reply: Final = await self.transport.eval(_POP_RECORD, (record_key(batch_id),), (_seconds(tombstone_ttl),))
|
||||
match reply:
|
||||
case RedisFailure() | None:
|
||||
|
|
@ -119,7 +168,8 @@ class TransportEnqueuedTokensStore:
|
|||
case b"":
|
||||
return AlreadyPopped()
|
||||
case bytes():
|
||||
return Popped(reply)
|
||||
decoded: Final = decode_reservation(reply)
|
||||
return decoded if isinstance(decoded, MalformedRecord) else Popped(decoded)
|
||||
case _:
|
||||
return _decode_failure("enqueued-token reservation record", reply)
|
||||
|
||||
|
|
|
|||
|
|
@ -171,8 +171,9 @@ def _redis_store(
|
|||
fail_from_call: Mapping[str, int] = MappingProxyType({}),
|
||||
raise_after_landing_keys: frozenset[str] = frozenset(),
|
||||
monotonic: Callable[[], float] = time.monotonic,
|
||||
namespace: str | None = None,
|
||||
) -> tuple[BatchEnqueuedTokenStore, fakeredis.FakeAsyncRedis]:
|
||||
cache = fake_redis_cache(server)
|
||||
cache = fake_redis_cache(server, namespace=namespace)
|
||||
raw = fakeredis.FakeAsyncRedis(server=server)
|
||||
litellm.in_memory_llm_clients_cache.set_cache(
|
||||
key=cache._get_async_client_cache_key(),
|
||||
|
|
@ -223,19 +224,54 @@ async def test_a_counter_an_old_pod_wrote_counts_against_the_allowance():
|
|||
assert await raw.get(_counter(scope)) == b"100"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_reservation_record_round_trips_with_old_pods():
|
||||
store, raw = _redis_store(fakeredis.FakeServer())
|
||||
scope = _scope(limit=100)
|
||||
reservation = BatchEnqueuedTokenReservation(tokens=5, scopes=(scope,), backend="redis")
|
||||
old_writer_document = _RESERVATION_ADAPTER.dump_json(reservation)
|
||||
await raw.set(_record("batch_old"), old_writer_document, ex=60)
|
||||
def _saved_reservation() -> BatchEnqueuedTokenReservation:
|
||||
return BatchEnqueuedTokenReservation(
|
||||
tokens=5,
|
||||
scopes=(
|
||||
BatchEnqueuedTokenScope(key="api_key", value='clé "one"', limit=100),
|
||||
BatchEnqueuedTokenScope(key="team", value="組/二", limit=200),
|
||||
),
|
||||
backend="memory",
|
||||
owner="worker\n雪",
|
||||
reserved_at_monotonic=1000.25,
|
||||
)
|
||||
|
||||
assert await store.pop_reservation("batch_old") == reservation
|
||||
assert await raw.get(_record("batch_old")) == b""
|
||||
|
||||
_SAVED_DOCUMENT: Final = (
|
||||
'{"tokens":5,"scopes":[{"key":"api_key","value":"clé \\"one\\"","limit":100},'
|
||||
'{"key":"team","value":"組/二","limit":200}],"backend":"memory",'
|
||||
'"owner":"worker\\n雪","reserved_at_monotonic":1000.25}'
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("namespace", (None, "test-namespace"))
|
||||
async def test_a_reservation_record_round_trips_with_old_pods(
|
||||
monkeypatch: pytest.MonkeyPatch, namespace: str | None
|
||||
) -> None:
|
||||
monkeypatch.setattr(time, "time", lambda: 1000.0)
|
||||
store, raw = _redis_store(fakeredis.FakeServer(), monotonic=lambda: 1030.5, namespace=namespace)
|
||||
reservation: Final = _saved_reservation()
|
||||
prefix: Final = f"{namespace}:" if namespace else ""
|
||||
old_key: Final = prefix + _record("batch_old")
|
||||
new_key: Final = prefix + _record("batch_new")
|
||||
await raw.set(old_key, _SAVED_DOCUMENT.encode("utf-8"), ex=60)
|
||||
|
||||
popped: Final = await store.pop_reservation("batch_old")
|
||||
assert popped == reservation
|
||||
assert popped.reserved_at_monotonic == reservation.reserved_at_monotonic
|
||||
assert await raw.get(old_key) == b""
|
||||
assert await raw.pttl(old_key) == BATCH_ENQUEUED_TOKEN_TTL_SECONDS * 1000
|
||||
assert await store.pop_reservation("batch_old") is None
|
||||
|
||||
await store.save_reservation("batch_new", reservation)
|
||||
assert _RESERVATION_ADAPTER.validate_json(await raw.get(_record("batch_new"))) == reservation
|
||||
document: Final = await raw.get(new_key)
|
||||
assert document == _SAVED_DOCUMENT.encode("utf-8")
|
||||
old_reader: Final = _RESERVATION_ADAPTER.validate_json(document)
|
||||
assert old_reader == reservation
|
||||
assert old_reader.reserved_at_monotonic == reservation.reserved_at_monotonic
|
||||
assert await raw.pttl(new_key) == (BATCH_ENQUEUED_TOKEN_TTL_SECONDS - 31) * 1000
|
||||
assert set(await raw.keys("*")) == {old_key.encode(), new_key.encode()}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -293,7 +329,9 @@ async def test_local_ghost_left_by_landed_save_never_refunds_twice():
|
|||
reservation = await store.reserve(tokens=60, scopes=(scope,))
|
||||
assert isinstance(reservation, BatchEnqueuedTokenReservation)
|
||||
await store.save_reservation("batch_ghost", reservation)
|
||||
assert await raw.get(_record("batch_ghost"))
|
||||
saved: Final = _RESERVATION_ADAPTER.dump_json(reservation)
|
||||
assert await raw.get(_record("batch_ghost")) == saved
|
||||
assert store.internal_usage_cache.dual_cache.in_memory_cache.cache_dict[_record("batch_ghost")] == saved.decode()
|
||||
|
||||
first = await store.pop_reservation("batch_ghost")
|
||||
assert first == reservation
|
||||
|
|
@ -313,6 +351,70 @@ async def test_local_ghost_left_by_landed_save_never_refunds_twice():
|
|||
assert await raw.get(_counter(scope)) == b"100"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("document", (b"{", b"{}", b"null", b'"text"', b"\xff"))
|
||||
async def test_consumed_malformed_record_does_not_refund_a_local_ghost(
|
||||
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, document: bytes
|
||||
) -> None:
|
||||
monkeypatch.setattr(time, "time", lambda: 1000.0)
|
||||
server: Final = fakeredis.FakeServer()
|
||||
store, raw = _redis_store(server, namespace="test-namespace")
|
||||
key: Final = _record("malformed")
|
||||
wire_key: Final = f"test-namespace:{key}"
|
||||
local: Final = store.internal_usage_cache.dual_cache.in_memory_cache
|
||||
local.set_cache(key=key, value=_SAVED_DOCUMENT)
|
||||
await raw.set(wire_key, document, ex=60)
|
||||
|
||||
assert await store.pop_reservation("malformed") is None
|
||||
assert "Discarding malformed enqueued-token reservation record for malformed" in caplog.text
|
||||
assert await raw.get(wire_key) == b""
|
||||
assert await raw.pttl(wire_key) == BATCH_ENQUEUED_TOKEN_TTL_SECONDS * 1000
|
||||
assert await store.pop_reservation("malformed") is None
|
||||
assert local.get_cache(key) is None
|
||||
server.connected = False
|
||||
assert await store.pop_reservation("malformed") is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reservation_record_outage_keeps_exact_local_bytes_and_consumes_once() -> None:
|
||||
server: Final = fakeredis.FakeServer()
|
||||
store, raw = _redis_store(server, monotonic=lambda: 1000.25, namespace="test-namespace")
|
||||
server.connected = False
|
||||
await store.save_reservation("outage", _saved_reservation())
|
||||
local: Final = store.internal_usage_cache.dual_cache.in_memory_cache
|
||||
assert local.cache_dict[_record("outage")] == _SAVED_DOCUMENT
|
||||
|
||||
popped: Final = await store.pop_reservation("outage")
|
||||
assert popped == _saved_reservation()
|
||||
assert popped.reserved_at_monotonic == 1000.25
|
||||
assert local.get_cache(_record("outage")) is None
|
||||
assert await store.pop_reservation("outage") is None
|
||||
server.connected = True
|
||||
assert await raw.keys("*") == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("raw", (_SAVED_DOCUMENT, _SAVED_DOCUMENT.encode("utf-8")))
|
||||
async def test_local_record_preserves_all_saved_fields(raw: str | bytes) -> None:
|
||||
store: Final = _in_memory_store()
|
||||
local: Final = store.internal_usage_cache.dual_cache.in_memory_cache
|
||||
local.set_cache(key=_record("local"), value=raw)
|
||||
popped: Final = await store.pop_reservation("local")
|
||||
assert popped == _saved_reservation()
|
||||
assert popped.reserved_at_monotonic == 1000.25
|
||||
assert local.get_cache(_record("local")) is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("raw", ("{", b"{}", {"tokens": 5}, 0))
|
||||
async def test_malformed_local_record_is_consumed_without_a_refund(raw: object) -> None:
|
||||
store: Final = _in_memory_store()
|
||||
local: Final = store.internal_usage_cache.dual_cache.in_memory_cache
|
||||
local.set_cache(key=_record("local"), value=raw)
|
||||
assert await store.pop_reservation("local") is None
|
||||
assert local.get_cache(_record("local")) is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_record_ttl_shrinks_by_elapsed_time_so_stale_records_never_outlive_their_counters():
|
||||
scope = _scope(limit=100)
|
||||
|
|
@ -360,16 +462,20 @@ async def test_failed_redis_refund_leaves_local_counters_untouched():
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pop_reservation_defaults_legacy_records_to_redis_backend():
|
||||
store = _in_memory_store()
|
||||
@pytest.mark.parametrize("redis_backed", (False, True))
|
||||
async def test_pop_reservation_defaults_legacy_records_to_redis_backend(redis_backed: bool) -> None:
|
||||
store, raw = _redis_store(fakeredis.FakeServer())
|
||||
legacy = '{"tokens": 5, "scopes": [{"key": "api_key", "value": "k", "limit": 10}]}'
|
||||
store.internal_usage_cache.dual_cache.in_memory_cache.set_cache(
|
||||
key="batch_enqueued_token_reservation:batch_legacy", value=legacy
|
||||
)
|
||||
if redis_backed:
|
||||
await raw.set(_record("batch_legacy"), legacy, ex=60)
|
||||
else:
|
||||
store.internal_usage_cache.dual_cache.in_memory_cache.set_cache(key=_record("batch_legacy"), value=legacy)
|
||||
popped = await store.pop_reservation("batch_legacy")
|
||||
assert popped == BatchEnqueuedTokenReservation(
|
||||
tokens=5, scopes=(BatchEnqueuedTokenScope(key="api_key", value="k", limit=10),), backend="redis"
|
||||
)
|
||||
assert popped.owner == ""
|
||||
assert isinstance(popped.reserved_at_monotonic, float)
|
||||
|
||||
|
||||
def test_canonical_provider_batch_id_passes_raw_ids_through():
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue