mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
refactor(redis-store): migrate batch enqueued-token accounting
Co-authored-by: Yujong Lee <yujong@berri.ai>
This commit is contained in:
parent
015aa2e3d1
commit
a6fd07c4d6
4 changed files with 332 additions and 275 deletions
|
|
@ -9,21 +9,31 @@ the reservation is refunded when the batch reaches a terminal state
|
|||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import math
|
||||
import time
|
||||
import uuid
|
||||
from collections.abc import Awaitable, Callable, Mapping, Sequence
|
||||
from collections.abc import Callable, Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING, Annotated, Final, Literal, Protocol, TypeAlias
|
||||
from datetime import timedelta
|
||||
from typing import TYPE_CHECKING, Annotated, Final, Literal, TypeAlias
|
||||
|
||||
from pydantic import ConfigDict, Field, TypeAdapter, ValidationError
|
||||
|
||||
from litellm._internal_context import with_service_target
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.caching.redis_cache import log_redis_failure
|
||||
from litellm.constants import BATCH_ENQUEUED_TOKEN_LIMIT_METADATA_KEY, BATCH_ENQUEUED_TOKEN_TTL_SECONDS
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.redis.stores.enqueued_tokens import (
|
||||
AlreadyPopped,
|
||||
EnqueuedTokensStore,
|
||||
OverLimit,
|
||||
Popped,
|
||||
Reserved,
|
||||
TransportEnqueuedTokensStore,
|
||||
counter_key,
|
||||
record_key,
|
||||
)
|
||||
from litellm.redis.transport.contract import RedisFailure
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -40,40 +50,6 @@ BATCH_ENQUEUED_REFUND_STATUSES: Final[frozenset[str]] = frozenset(
|
|||
|
||||
ScopeKey: TypeAlias = Literal["api_key", "team"]
|
||||
|
||||
RESERVE_ENQUEUED_TOKENS_SCRIPT: Final = """
|
||||
local amount = tonumber(ARGV[1])
|
||||
local ttl = tonumber(ARGV[2])
|
||||
local limit = tonumber(ARGV[3])
|
||||
local current = tonumber(redis.call('GET', KEYS[1]) or '0')
|
||||
if current + amount > limit then
|
||||
return {0, current}
|
||||
end
|
||||
local updated = redis.call('INCRBY', KEYS[1], amount)
|
||||
redis.call('EXPIRE', KEYS[1], ttl)
|
||||
return {1, updated}
|
||||
"""
|
||||
|
||||
REFUND_ENQUEUED_TOKENS_SCRIPT: Final = """
|
||||
local updated = redis.call('DECRBY', KEYS[1], tonumber(ARGV[1]))
|
||||
if updated <= 0 then
|
||||
redis.call('DEL', KEYS[1])
|
||||
end
|
||||
return 1
|
||||
"""
|
||||
|
||||
SAVE_RESERVATION_SCRIPT: Final = """
|
||||
redis.call('SET', KEYS[1], ARGV[1], 'EX', tonumber(ARGV[2]))
|
||||
return 1
|
||||
"""
|
||||
|
||||
POP_RESERVATION_SCRIPT: Final = """
|
||||
local value = redis.call('GET', KEYS[1])
|
||||
if value and value ~= '' then
|
||||
redis.call('SET', KEYS[1], '', 'EX', tonumber(ARGV[1]))
|
||||
end
|
||||
return value
|
||||
"""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class BatchEnqueuedTokenScope:
|
||||
|
|
@ -103,17 +79,11 @@ class BatchEnqueuedTokenOverLimit:
|
|||
BatchEnqueuedTokenOutcome: TypeAlias = BatchEnqueuedTokenReservation | BatchEnqueuedTokenOverLimit
|
||||
|
||||
_LIMIT_ADAPTER: Final = TypeAdapter(Annotated[int, Field(gt=0)])
|
||||
_RESERVE_RESULT_ADAPTER: Final = TypeAdapter(tuple[int, int])
|
||||
_POPPED_VALUE_ADAPTER: Final = TypeAdapter(str | bytes | None)
|
||||
_STORED_COUNTER_ADAPTER: Final = TypeAdapter(int | None)
|
||||
_RESERVATION_ADAPTER: Final = TypeAdapter(BatchEnqueuedTokenReservation)
|
||||
_LOCAL_RECORD_ADAPTER: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
class _ScriptRunner(Protocol):
|
||||
def __call__(self, keys: Sequence[str], args: Sequence[str | bytes | int | float]) -> Awaitable[object]: ...
|
||||
|
||||
|
||||
def _read_metadata_limit(metadata: Mapping[str, object] | None) -> int | None:
|
||||
if not metadata:
|
||||
return None
|
||||
|
|
@ -202,27 +172,18 @@ class BatchEnqueuedTokenStore:
|
|||
self._monotonic: Final = monotonic
|
||||
self._lock = asyncio.Lock()
|
||||
self._owner_token = uuid.uuid4().hex
|
||||
redis_cache = internal_usage_cache.dual_cache.redis_cache
|
||||
self._reserve_script: _ScriptRunner | None = (
|
||||
redis_cache.async_register_script(RESERVE_ENQUEUED_TOKENS_SCRIPT) if redis_cache is not None else None
|
||||
)
|
||||
self._refund_script: _ScriptRunner | None = (
|
||||
redis_cache.async_register_script(REFUND_ENQUEUED_TOKENS_SCRIPT) if redis_cache is not None else None
|
||||
)
|
||||
self._save_script: _ScriptRunner | None = (
|
||||
redis_cache.async_register_script(SAVE_RESERVATION_SCRIPT) if redis_cache is not None else None
|
||||
)
|
||||
self._pop_script: _ScriptRunner | None = (
|
||||
redis_cache.async_register_script(POP_RESERVATION_SCRIPT) if redis_cache is not None else None
|
||||
redis_cache: Final = internal_usage_cache.dual_cache.redis_cache
|
||||
self._store: Final[EnqueuedTokensStore | None] = (
|
||||
TransportEnqueuedTokensStore(redis_cache.transport()) if redis_cache is not None else None
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _counter_key(scope: BatchEnqueuedTokenScope) -> str:
|
||||
return f"batch_enqueued_tokens:{scope.key}:{scope.value}"
|
||||
return counter_key(scope.key, scope.value)
|
||||
|
||||
@staticmethod
|
||||
def _record_key(batch_id: str) -> str:
|
||||
return f"batch_enqueued_token_reservation:{batch_id}"
|
||||
return record_key(batch_id)
|
||||
|
||||
@with_service_target("rate_limits")
|
||||
async def reserve(
|
||||
|
|
@ -233,83 +194,58 @@ class BatchEnqueuedTokenStore:
|
|||
) -> BatchEnqueuedTokenOutcome:
|
||||
if tokens <= 0 or not scopes:
|
||||
return BatchEnqueuedTokenReservation(tokens=max(tokens, 0), scopes=scopes)
|
||||
reserve_script: Final = self._reserve_script
|
||||
refund_script: Final = self._refund_script
|
||||
if reserve_script is not None and refund_script is not None:
|
||||
try:
|
||||
return await self._reserve_via_redis(reserve_script, refund_script, tokens=tokens, scopes=scopes)
|
||||
except Exception as e: # noqa: BLE001 # any Redis failure must fall back to the in-memory counters
|
||||
log_redis_failure(
|
||||
verbose_proxy_logger,
|
||||
logging.WARNING,
|
||||
"Redis enqueued-token reserve failed, falling back to in-memory",
|
||||
e,
|
||||
)
|
||||
if self._store is not None:
|
||||
outcome: Final = await self._reserve_via_redis(self._store, tokens=tokens, scopes=scopes)
|
||||
if not isinstance(outcome, RedisFailure):
|
||||
return outcome
|
||||
return await self._reserve_in_memory(tokens=tokens, scopes=scopes, span=litellm_parent_otel_span)
|
||||
|
||||
async def _reserve_via_redis(
|
||||
self,
|
||||
reserve_script: _ScriptRunner,
|
||||
refund_script: _ScriptRunner,
|
||||
store: EnqueuedTokensStore,
|
||||
tokens: int,
|
||||
scopes: tuple[BatchEnqueuedTokenScope, ...],
|
||||
) -> BatchEnqueuedTokenOutcome:
|
||||
) -> BatchEnqueuedTokenOutcome | RedisFailure:
|
||||
started: Final = self._monotonic()
|
||||
ttl: Final = timedelta(seconds=BATCH_ENQUEUED_TOKEN_TTL_SECONDS)
|
||||
for index, scope in enumerate(scopes):
|
||||
result = await self._run_reserve_script(
|
||||
reserve_script,
|
||||
refund_script,
|
||||
tokens=tokens,
|
||||
scope=scope,
|
||||
already_reserved=scopes[:index],
|
||||
)
|
||||
if result[0] != 1:
|
||||
await self._rollback_partial_reserve(refund_script, tokens=tokens, scopes=scopes[:index])
|
||||
return BatchEnqueuedTokenOverLimit(scope=scope, enqueued=result[1])
|
||||
reply = await store.reserve(scope.key, scope.value, tokens, limit=scope.limit, ttl=ttl)
|
||||
match reply:
|
||||
case Reserved():
|
||||
continue
|
||||
case OverLimit(enqueued=enqueued):
|
||||
await self._rollback_partial_reserve(store, tokens=tokens, scopes=scopes[:index])
|
||||
return BatchEnqueuedTokenOverLimit(scope=scope, enqueued=enqueued)
|
||||
case RedisFailure():
|
||||
await self._rollback_partial_reserve(store, tokens=tokens, scopes=scopes[:index])
|
||||
return reply
|
||||
return BatchEnqueuedTokenReservation(
|
||||
tokens=tokens, scopes=scopes, backend="redis", reserved_at_monotonic=started
|
||||
)
|
||||
|
||||
async def _run_reserve_script(
|
||||
self,
|
||||
reserve_script: _ScriptRunner,
|
||||
refund_script: _ScriptRunner,
|
||||
tokens: int,
|
||||
scope: BatchEnqueuedTokenScope,
|
||||
already_reserved: tuple[BatchEnqueuedTokenScope, ...],
|
||||
) -> tuple[int, int]:
|
||||
try:
|
||||
raw_result: Final = await reserve_script(
|
||||
(self._counter_key(scope),),
|
||||
(tokens, BATCH_ENQUEUED_TOKEN_TTL_SECONDS, scope.limit),
|
||||
)
|
||||
return _RESERVE_RESULT_ADAPTER.validate_python(raw_result)
|
||||
except Exception:
|
||||
await self._rollback_partial_reserve(refund_script, tokens=tokens, scopes=already_reserved)
|
||||
raise
|
||||
|
||||
async def _rollback_partial_reserve(
|
||||
self,
|
||||
refund_script: _ScriptRunner,
|
||||
store: EnqueuedTokensStore,
|
||||
tokens: int,
|
||||
scopes: tuple[BatchEnqueuedTokenScope, ...],
|
||||
) -> None:
|
||||
try:
|
||||
await self._refund_via_redis(refund_script, tokens=tokens, scopes=scopes)
|
||||
except Exception as e: # noqa: BLE001 # best-effort rollback: the leak is TTL-bounded and only tightens the allowance
|
||||
refunded: Final = await self._refund_via_redis(store, tokens=tokens, scopes=scopes)
|
||||
if isinstance(refunded, RedisFailure):
|
||||
verbose_proxy_logger.warning(
|
||||
"Rollback of partially reserved enqueued tokens failed; leaked increments expire with the TTL: %s",
|
||||
str(e),
|
||||
"Rollback of partially reserved enqueued tokens failed; leaked increments expire with the TTL"
|
||||
)
|
||||
|
||||
async def _refund_via_redis(
|
||||
self,
|
||||
refund_script: _ScriptRunner,
|
||||
store: EnqueuedTokensStore,
|
||||
tokens: int,
|
||||
scopes: tuple[BatchEnqueuedTokenScope, ...],
|
||||
) -> None:
|
||||
) -> None | RedisFailure:
|
||||
for scope in scopes:
|
||||
await refund_script((self._counter_key(scope),), (tokens,))
|
||||
refunded = await store.refund(scope.key, scope.value, tokens)
|
||||
if refunded is not None:
|
||||
return refunded
|
||||
return None
|
||||
|
||||
async def _reserve_in_memory(
|
||||
self,
|
||||
|
|
@ -355,18 +291,16 @@ class BatchEnqueuedTokenStore:
|
|||
await self._set_local_counter(scope, remaining, litellm_parent_otel_span)
|
||||
|
||||
async def _refund_redis_reservation(self, reservation: BatchEnqueuedTokenReservation) -> None:
|
||||
refund_script: Final = self._refund_script
|
||||
if refund_script is None:
|
||||
if self._store is None:
|
||||
verbose_proxy_logger.warning(
|
||||
"No Redis client for a Redis-granted enqueued-token refund; leaked increments expire with the TTL"
|
||||
)
|
||||
return
|
||||
try:
|
||||
await self._refund_via_redis(refund_script, tokens=reservation.tokens, scopes=reservation.scopes)
|
||||
except Exception as e: # noqa: BLE001 # best-effort refund: the leak is TTL-bounded and only tightens the allowance
|
||||
verbose_proxy_logger.warning(
|
||||
"Redis enqueued-token refund failed; leaked increments expire with the TTL: %s", str(e)
|
||||
)
|
||||
refunded: Final = await self._refund_via_redis(
|
||||
self._store, tokens=reservation.tokens, scopes=reservation.scopes
|
||||
)
|
||||
if isinstance(refunded, RedisFailure):
|
||||
verbose_proxy_logger.warning("Redis enqueued-token refund failed; leaked increments expire with the TTL")
|
||||
|
||||
@with_service_target("rate_limits")
|
||||
async def save_reservation(
|
||||
|
|
@ -378,20 +312,9 @@ class BatchEnqueuedTokenStore:
|
|||
serialized: Final = _RESERVATION_ADAPTER.dump_json(reservation).decode("utf-8")
|
||||
elapsed: Final = self._monotonic() - reservation.reserved_at_monotonic
|
||||
ttl: Final = max(1, BATCH_ENQUEUED_TOKEN_TTL_SECONDS - math.ceil(elapsed))
|
||||
if self._save_script is not None:
|
||||
try:
|
||||
await self._save_script(
|
||||
(self._record_key(batch_id),),
|
||||
(serialized, ttl),
|
||||
)
|
||||
except Exception as e: # noqa: BLE001 # any Redis failure must fall back to the in-memory record
|
||||
log_redis_failure(
|
||||
verbose_proxy_logger,
|
||||
logging.WARNING,
|
||||
"Redis enqueued-token reservation save failed, falling back to in-memory",
|
||||
e,
|
||||
)
|
||||
else:
|
||||
if self._store is not None:
|
||||
saved: Final = await self._store.save_record(batch_id, serialized, ttl=timedelta(seconds=ttl))
|
||||
if saved is None:
|
||||
return
|
||||
await self.internal_usage_cache.async_set_cache(
|
||||
key=self._record_key(batch_id),
|
||||
|
|
@ -407,15 +330,17 @@ class BatchEnqueuedTokenStore:
|
|||
batch_id: str,
|
||||
litellm_parent_otel_span: "Span | None" = None,
|
||||
) -> BatchEnqueuedTokenReservation | None:
|
||||
redis_raw: Final = await self._pop_redis_record(batch_id)
|
||||
if redis_raw is not None and not redis_raw:
|
||||
popped: Final = await self._pop_redis_record(batch_id)
|
||||
if isinstance(popped, AlreadyPopped):
|
||||
# The Redis pop tombstones popped records in place, so a hit on the empty
|
||||
# tombstone means the batch was already refunded elsewhere; a local copy
|
||||
# left behind by a save that raised after landing must not refund again.
|
||||
await self._pop_local_record(batch_id, litellm_parent_otel_span)
|
||||
return None
|
||||
raw: Final = (
|
||||
redis_raw if redis_raw is not None else await self._pop_local_record(batch_id, litellm_parent_otel_span)
|
||||
popped.document
|
||||
if isinstance(popped, Popped)
|
||||
else await self._pop_local_record(batch_id, litellm_parent_otel_span)
|
||||
)
|
||||
if raw is None:
|
||||
return None
|
||||
|
|
@ -427,22 +352,13 @@ class BatchEnqueuedTokenStore:
|
|||
verbose_proxy_logger.warning("Discarding malformed enqueued-token reservation record for %s", batch_id)
|
||||
return None
|
||||
|
||||
async def _pop_redis_record(self, batch_id: str) -> str | bytes | None:
|
||||
pop_script: Final = self._pop_script
|
||||
if pop_script is None:
|
||||
return None
|
||||
try:
|
||||
return _POPPED_VALUE_ADAPTER.validate_python(
|
||||
await pop_script((self._record_key(batch_id),), (BATCH_ENQUEUED_TOKEN_TTL_SECONDS,))
|
||||
)
|
||||
except Exception as e: # noqa: BLE001 # any Redis failure must fall back to the in-memory record
|
||||
log_redis_failure(
|
||||
verbose_proxy_logger,
|
||||
logging.WARNING,
|
||||
"Redis enqueued-token reservation pop failed, falling back to in-memory",
|
||||
e,
|
||||
)
|
||||
async def _pop_redis_record(self, batch_id: str) -> Popped | AlreadyPopped | None:
|
||||
if self._store is None:
|
||||
return None
|
||||
popped: Final = await self._store.pop_record(
|
||||
batch_id, tombstone_ttl=timedelta(seconds=BATCH_ENQUEUED_TOKEN_TTL_SECONDS)
|
||||
)
|
||||
return None if isinstance(popped, RedisFailure) else popped
|
||||
|
||||
async def _pop_local_record(self, batch_id: str, span: "Span | None") -> object:
|
||||
async with self._lock:
|
||||
|
|
|
|||
132
litellm/redis/stores/enqueued_tokens.py
Normal file
132
litellm/redis/stores/enqueued_tokens.py
Normal file
|
|
@ -0,0 +1,132 @@
|
|||
from dataclasses import dataclass
|
||||
from datetime import timedelta
|
||||
from typing import Final, Protocol
|
||||
|
||||
from litellm.redis.transport.contract import EvalReply, RedisFailure, RedisTransport
|
||||
|
||||
_RESERVE: Final = """
|
||||
local amount = tonumber(ARGV[1])
|
||||
local ttl = tonumber(ARGV[2])
|
||||
local limit = tonumber(ARGV[3])
|
||||
local current = tonumber(redis.call('GET', KEYS[1]) or '0')
|
||||
if current + amount > limit then
|
||||
return {0, current}
|
||||
end
|
||||
local updated = redis.call('INCRBY', KEYS[1], amount)
|
||||
redis.call('EXPIRE', KEYS[1], ttl)
|
||||
return {1, updated}
|
||||
"""
|
||||
|
||||
_REFUND: Final = """
|
||||
local updated = redis.call('DECRBY', KEYS[1], tonumber(ARGV[1]))
|
||||
if updated <= 0 then
|
||||
redis.call('DEL', KEYS[1])
|
||||
end
|
||||
return 1
|
||||
"""
|
||||
|
||||
_SAVE_RECORD: Final = """
|
||||
redis.call('SET', KEYS[1], ARGV[1], 'EX', tonumber(ARGV[2]))
|
||||
return 1
|
||||
"""
|
||||
|
||||
_POP_RECORD: Final = """
|
||||
local value = redis.call('GET', KEYS[1])
|
||||
if value and value ~= '' then
|
||||
redis.call('SET', KEYS[1], '', 'EX', tonumber(ARGV[1]))
|
||||
end
|
||||
return value
|
||||
"""
|
||||
|
||||
|
||||
def counter_key(scope_key: str, scope_value: str) -> str:
|
||||
return f"batch_enqueued_tokens:{scope_key}:{scope_value}"
|
||||
|
||||
|
||||
def record_key(batch_id: str) -> str:
|
||||
return f"batch_enqueued_token_reservation:{batch_id}"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Reserved:
|
||||
enqueued: int
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class OverLimit:
|
||||
enqueued: int
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Popped:
|
||||
document: bytes
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class AlreadyPopped:
|
||||
pass
|
||||
|
||||
|
||||
class EnqueuedTokensStore(Protocol):
|
||||
async def reserve(
|
||||
self, scope_key: str, scope_value: str, tokens: int, *, limit: int, ttl: timedelta
|
||||
) -> Reserved | OverLimit | RedisFailure: ...
|
||||
|
||||
async def refund(self, scope_key: str, scope_value: str, tokens: int) -> None | RedisFailure: ...
|
||||
|
||||
async def save_record(self, batch_id: str, document: str, *, ttl: timedelta) -> None | RedisFailure: ...
|
||||
|
||||
async def pop_record(
|
||||
self, batch_id: str, *, tombstone_ttl: timedelta
|
||||
) -> Popped | AlreadyPopped | None | RedisFailure: ...
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class TransportEnqueuedTokensStore:
|
||||
transport: RedisTransport
|
||||
|
||||
async def reserve(
|
||||
self, scope_key: str, scope_value: str, tokens: int, *, limit: int, ttl: timedelta
|
||||
) -> Reserved | OverLimit | RedisFailure:
|
||||
reply: Final = await self.transport.eval(
|
||||
_RESERVE, (counter_key(scope_key, scope_value),), (tokens, _seconds(ttl), limit)
|
||||
)
|
||||
match reply:
|
||||
case RedisFailure():
|
||||
return reply
|
||||
case (1, int(enqueued)):
|
||||
return Reserved(enqueued)
|
||||
case (0, int(enqueued)):
|
||||
return OverLimit(enqueued)
|
||||
case _:
|
||||
return _decode_failure("enqueued-token reserve", reply)
|
||||
|
||||
async def refund(self, scope_key: str, scope_value: str, tokens: int) -> None | RedisFailure:
|
||||
reply: Final = await self.transport.eval(_REFUND, (counter_key(scope_key, scope_value),), (tokens,))
|
||||
return reply if isinstance(reply, RedisFailure) else None
|
||||
|
||||
async def save_record(self, batch_id: str, document: str, *, ttl: timedelta) -> None | RedisFailure:
|
||||
reply: Final = await self.transport.eval(_SAVE_RECORD, (record_key(batch_id),), (document, _seconds(ttl)))
|
||||
return reply if isinstance(reply, RedisFailure) else None
|
||||
|
||||
async def pop_record(
|
||||
self, batch_id: str, *, tombstone_ttl: timedelta
|
||||
) -> Popped | AlreadyPopped | None | RedisFailure:
|
||||
reply: Final = await self.transport.eval(_POP_RECORD, (record_key(batch_id),), (_seconds(tombstone_ttl),))
|
||||
match reply:
|
||||
case RedisFailure() | None:
|
||||
return reply
|
||||
case b"":
|
||||
return AlreadyPopped()
|
||||
case bytes():
|
||||
return Popped(reply)
|
||||
case _:
|
||||
return _decode_failure("enqueued-token reservation record", reply)
|
||||
|
||||
|
||||
def _seconds(ttl: timedelta) -> int:
|
||||
return ttl // timedelta(seconds=1)
|
||||
|
||||
|
||||
def _decode_failure(what: str, reply: EvalReply) -> RedisFailure:
|
||||
return RedisFailure(kind="decode", message=f"unexpected {what} reply: {reply!r}")
|
||||
|
|
@ -8,18 +8,25 @@ response-shape helpers the v3 limiter's post-call hooks rely on.
|
|||
|
||||
import base64
|
||||
import logging
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
from collections.abc import Mapping, Sequence
|
||||
from collections.abc import Callable, Mapping
|
||||
from types import MappingProxyType, SimpleNamespace
|
||||
from typing import Final
|
||||
|
||||
import fakeredis
|
||||
import pytest
|
||||
from redis.exceptions import ConnectionError as RedisConnectionError
|
||||
from redis.exceptions import NoScriptError
|
||||
from redis.exceptions import TimeoutError as RedisTimeoutError
|
||||
|
||||
import litellm
|
||||
from litellm.caching.caching import DualCache
|
||||
from litellm.caching.redis_cache import RedisCircuitBreakerOpenError
|
||||
from litellm.constants import BATCH_ENQUEUED_TOKEN_TTL_SECONDS
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.hooks.batch_enqueued_tokens import (
|
||||
_RESERVATION_ADAPTER,
|
||||
BatchEnqueuedTokenOverLimit,
|
||||
BatchEnqueuedTokenReservation,
|
||||
BatchEnqueuedTokenScope,
|
||||
|
|
@ -29,6 +36,7 @@ from litellm.proxy.hooks.batch_enqueued_tokens import (
|
|||
resolve_batch_enqueued_token_scopes,
|
||||
)
|
||||
from litellm.proxy.utils import InternalUsageCache
|
||||
from tests._support.fake_redis import fake_redis_cache
|
||||
|
||||
|
||||
def _in_memory_store() -> BatchEnqueuedTokenStore:
|
||||
|
|
@ -127,113 +135,121 @@ async def test_zero_token_reserve_charges_nothing():
|
|||
assert isinstance(full, BatchEnqueuedTokenReservation)
|
||||
|
||||
|
||||
class _SingleKeyRedisFake:
|
||||
"""Emulates the Redis script path one single-key call at a time, recording every call."""
|
||||
|
||||
class _FaultyClient:
|
||||
def __init__(
|
||||
self,
|
||||
fail_reserve_keys: frozenset[str] = frozenset(),
|
||||
fail_refund_keys: frozenset[str] = frozenset(),
|
||||
fail_save_keys: frozenset[str] = frozenset(),
|
||||
raise_after_landing_save_keys: frozenset[str] = frozenset(),
|
||||
inner: fakeredis.FakeAsyncRedis,
|
||||
fail_from_call: Mapping[str, int],
|
||||
raise_after_landing_keys: frozenset[str],
|
||||
) -> None:
|
||||
self.script_calls: tuple[tuple[str, tuple[str, ...]], ...] = ()
|
||||
self.save_ttls: tuple[int, ...] = ()
|
||||
self.counters: Mapping[str, int] = MappingProxyType({})
|
||||
self.records: Mapping[str, str] = MappingProxyType({})
|
||||
self.fail_reserve_keys = fail_reserve_keys
|
||||
self.fail_refund_keys = fail_refund_keys
|
||||
self.fail_save_keys = fail_save_keys
|
||||
self.raise_after_landing_save_keys = raise_after_landing_save_keys
|
||||
self.inner = inner
|
||||
self.fail_from_call = fail_from_call
|
||||
self.raise_after_landing_keys = raise_after_landing_keys
|
||||
self.calls: Mapping[str, int] = MappingProxyType({})
|
||||
|
||||
def async_register_script(self, script: str):
|
||||
kind: Final = (
|
||||
"reserve"
|
||||
if "INCRBY" in script
|
||||
else "refund" if "DECRBY" in script else "pop" if "GET" in script else "save"
|
||||
)
|
||||
async def evalsha(self, sha: str, numkeys: int, *keys_and_args: str | bytes | float) -> object:
|
||||
if not (await self.inner.script_exists(sha))[0]:
|
||||
raise NoScriptError("No matching script")
|
||||
return await self._run(self.inner.evalsha, sha, numkeys, keys_and_args)
|
||||
|
||||
async def run(keys: Sequence[str], args: Sequence[str | bytes | int | float]) -> object:
|
||||
self.script_calls = (*self.script_calls, (kind, tuple(keys)))
|
||||
return self._run(kind, tuple(keys), tuple(args))
|
||||
async def eval(self, script: str, numkeys: int, *keys_and_args: str | bytes | float) -> object:
|
||||
return await self._run(self.inner.eval, script, numkeys, keys_and_args)
|
||||
|
||||
return run
|
||||
async def _run(self, run, script: str, numkeys: int, keys_and_args: tuple[str | bytes | int | float, ...]):
|
||||
key: Final = str(keys_and_args[0])
|
||||
self.calls = MappingProxyType({**self.calls, key: self.calls.get(key, 0) + 1})
|
||||
if self.calls[key] >= self.fail_from_call.get(key, sys.maxsize):
|
||||
raise RedisConnectionError(f"simulated redis failure for {key}")
|
||||
reply: Final = await run(script, numkeys, *keys_and_args)
|
||||
if key in self.raise_after_landing_keys:
|
||||
raise RedisTimeoutError(f"simulated redis timeout after landing for {key}")
|
||||
return reply
|
||||
|
||||
def _run(self, kind: str, keys: tuple[str, ...], args: tuple[str | bytes | int | float, ...]) -> object:
|
||||
if kind == "reserve":
|
||||
if keys[0] in self.fail_reserve_keys:
|
||||
raise ConnectionError(f"simulated redis failure for {keys[0]}")
|
||||
amount, limit = int(args[0]), int(args[2])
|
||||
current: Final = self.counters.get(keys[0], 0)
|
||||
if current + amount > limit:
|
||||
return (0, current)
|
||||
self.counters = MappingProxyType({**self.counters, keys[0]: current + amount})
|
||||
return (1, current + amount)
|
||||
if kind == "refund":
|
||||
if keys[0] in self.fail_refund_keys:
|
||||
raise ConnectionError(f"simulated redis failure for {keys[0]}")
|
||||
remaining: Final = self.counters.get(keys[0], 0) - int(args[0])
|
||||
self.counters = MappingProxyType(
|
||||
{key: value for key, value in self.counters.items() if key != keys[0]}
|
||||
if remaining <= 0
|
||||
else {**self.counters, keys[0]: remaining}
|
||||
)
|
||||
return 1
|
||||
if kind == "save":
|
||||
if keys[0] in self.fail_save_keys:
|
||||
raise ConnectionError(f"simulated redis failure for {keys[0]}")
|
||||
self.records = MappingProxyType({**self.records, keys[0]: str(args[0])})
|
||||
self.save_ttls = (*self.save_ttls, int(args[1]))
|
||||
if keys[0] in self.raise_after_landing_save_keys:
|
||||
raise TimeoutError(f"simulated redis timeout after landing for {keys[0]}")
|
||||
return 1
|
||||
if kind == "pop":
|
||||
popped: Final = self.records.get(keys[0])
|
||||
if popped:
|
||||
self.records = MappingProxyType({**self.records, keys[0]: ""})
|
||||
return popped
|
||||
raise AssertionError(f"unexpected {kind} script call for keys {keys}")
|
||||
|
||||
def _redis_store(
|
||||
server: fakeredis.FakeServer,
|
||||
fail_from_call: Mapping[str, int] = MappingProxyType({}),
|
||||
raise_after_landing_keys: frozenset[str] = frozenset(),
|
||||
monotonic: Callable[[], float] = time.monotonic,
|
||||
) -> tuple[BatchEnqueuedTokenStore, fakeredis.FakeAsyncRedis]:
|
||||
cache = fake_redis_cache(server)
|
||||
raw = fakeredis.FakeAsyncRedis(server=server)
|
||||
litellm.in_memory_llm_clients_cache.set_cache(
|
||||
key=cache._get_async_client_cache_key(),
|
||||
value=_FaultyClient(raw, fail_from_call=fail_from_call, raise_after_landing_keys=raise_after_landing_keys),
|
||||
)
|
||||
store = BatchEnqueuedTokenStore(
|
||||
internal_usage_cache=InternalUsageCache(DualCache(redis_cache=cache, default_in_memory_ttl=60)),
|
||||
monotonic=monotonic,
|
||||
)
|
||||
return store, raw
|
||||
|
||||
|
||||
def _counter(scope: BatchEnqueuedTokenScope) -> str:
|
||||
return f"batch_enqueued_tokens:{scope.key}:{scope.value}"
|
||||
|
||||
|
||||
def _record(batch_id: str) -> str:
|
||||
return f"batch_enqueued_token_reservation:{batch_id}"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_redis_reserve_issues_single_key_calls_and_rolls_back_on_over_limit():
|
||||
fake = _SingleKeyRedisFake()
|
||||
store = BatchEnqueuedTokenStore(
|
||||
internal_usage_cache=InternalUsageCache(DualCache(redis_cache=fake, default_in_memory_ttl=60))
|
||||
)
|
||||
async def test_redis_reserve_rolls_back_on_over_limit_and_refund_clears_the_counters():
|
||||
store, raw = _redis_store(fakeredis.FakeServer())
|
||||
key_scope = _scope(limit=100, key="api_key")
|
||||
team_scope = _scope(limit=50, key="team")
|
||||
|
||||
over = await store.reserve(tokens=60, scopes=(key_scope, team_scope))
|
||||
assert over == BatchEnqueuedTokenOverLimit(scope=team_scope, enqueued=0)
|
||||
assert tuple(kind for kind, _ in fake.script_calls) == ("reserve", "reserve", "refund")
|
||||
assert not fake.counters
|
||||
assert await raw.keys("*") == []
|
||||
|
||||
fits = await store.reserve(tokens=50, scopes=(key_scope, team_scope))
|
||||
assert isinstance(fits, BatchEnqueuedTokenReservation)
|
||||
assert fits.backend == "redis"
|
||||
assert await raw.get(_counter(key_scope)) == b"50"
|
||||
assert 0 < await raw.pttl(_counter(team_scope)) <= BATCH_ENQUEUED_TOKEN_TTL_SECONDS * 1000
|
||||
await store.refund(fits)
|
||||
assert not fake.counters
|
||||
assert all(len(keys) == 1 for _, keys in fake.script_calls)
|
||||
assert await raw.keys("*") == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_counter_an_old_pod_wrote_counts_against_the_allowance():
|
||||
store, raw = _redis_store(fakeredis.FakeServer())
|
||||
scope = _scope(limit=100)
|
||||
await raw.set(_counter(scope), 70, ex=BATCH_ENQUEUED_TOKEN_TTL_SECONDS)
|
||||
|
||||
assert await store.reserve(tokens=40, scopes=(scope,)) == BatchEnqueuedTokenOverLimit(scope=scope, enqueued=70)
|
||||
assert isinstance(await store.reserve(tokens=30, scopes=(scope,)), BatchEnqueuedTokenReservation)
|
||||
assert await raw.get(_counter(scope)) == b"100"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_reservation_record_round_trips_with_old_pods():
|
||||
store, raw = _redis_store(fakeredis.FakeServer())
|
||||
scope = _scope(limit=100)
|
||||
reservation = BatchEnqueuedTokenReservation(tokens=5, scopes=(scope,), backend="redis")
|
||||
old_writer_document = _RESERVATION_ADAPTER.dump_json(reservation)
|
||||
await raw.set(_record("batch_old"), old_writer_document, ex=60)
|
||||
|
||||
assert await store.pop_reservation("batch_old") == reservation
|
||||
assert await raw.get(_record("batch_old")) == b""
|
||||
|
||||
await store.save_reservation("batch_new", reservation)
|
||||
assert _RESERVATION_ADAPTER.validate_json(await raw.get(_record("batch_new"))) == reservation
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_partial_redis_reserve_failure_rolls_back_and_grants_in_memory():
|
||||
key_scope = _scope(limit=100, key="api_key")
|
||||
team_scope = _scope(limit=50, key="team")
|
||||
fake = _SingleKeyRedisFake(fail_reserve_keys=frozenset({f"batch_enqueued_tokens:team:{team_scope.value}"}))
|
||||
store = BatchEnqueuedTokenStore(
|
||||
internal_usage_cache=InternalUsageCache(DualCache(redis_cache=fake, default_in_memory_ttl=60))
|
||||
)
|
||||
store, raw = _redis_store(fakeredis.FakeServer(), fail_from_call=MappingProxyType({_counter(team_scope): 1}))
|
||||
|
||||
outcome = await store.reserve(tokens=10, scopes=(key_scope, team_scope))
|
||||
assert isinstance(outcome, BatchEnqueuedTokenReservation)
|
||||
assert outcome.backend == "memory"
|
||||
assert tuple(kind for kind, _ in fake.script_calls) == ("reserve", "reserve", "refund")
|
||||
assert not fake.counters
|
||||
assert await raw.exists(_counter(key_scope)) == 0
|
||||
|
||||
await store.refund(outcome)
|
||||
assert tuple(kind for kind, _ in fake.script_calls) == ("reserve", "reserve", "refund")
|
||||
|
||||
refilled = await store.reserve(tokens=50, scopes=(team_scope,))
|
||||
assert isinstance(refilled, BatchEnqueuedTokenReservation)
|
||||
assert refilled.backend == "memory"
|
||||
|
|
@ -243,84 +259,70 @@ async def test_partial_redis_reserve_failure_rolls_back_and_grants_in_memory():
|
|||
async def test_over_limit_verdict_survives_a_failing_rollback():
|
||||
key_scope = _scope(limit=100, key="api_key")
|
||||
team_scope = _scope(limit=5, key="team")
|
||||
key_counter: Final = f"batch_enqueued_tokens:api_key:{key_scope.value}"
|
||||
fake = _SingleKeyRedisFake(fail_refund_keys=frozenset({key_counter}))
|
||||
store = BatchEnqueuedTokenStore(
|
||||
internal_usage_cache=InternalUsageCache(DualCache(redis_cache=fake, default_in_memory_ttl=60))
|
||||
)
|
||||
store, raw = _redis_store(fakeredis.FakeServer(), fail_from_call=MappingProxyType({_counter(key_scope): 2}))
|
||||
|
||||
outcome = await store.reserve(tokens=10, scopes=(key_scope, team_scope))
|
||||
assert outcome == BatchEnqueuedTokenOverLimit(scope=team_scope, enqueued=0)
|
||||
assert fake.counters == {key_counter: 10}
|
||||
assert await raw.get(_counter(key_scope)) == b"10"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pop_falls_back_to_local_record_when_redis_pop_finds_nothing():
|
||||
async def test_pop_falls_back_to_local_record_when_the_redis_save_failed():
|
||||
scope = _scope(limit=100)
|
||||
record_key: Final = "batch_enqueued_token_reservation:batch_local_record"
|
||||
fake = _SingleKeyRedisFake(fail_save_keys=frozenset({record_key}))
|
||||
store = BatchEnqueuedTokenStore(
|
||||
internal_usage_cache=InternalUsageCache(DualCache(redis_cache=fake, default_in_memory_ttl=60))
|
||||
store, raw = _redis_store(
|
||||
fakeredis.FakeServer(), fail_from_call=MappingProxyType({_record("batch_local_record"): 1})
|
||||
)
|
||||
|
||||
reservation = await store.reserve(tokens=60, scopes=(scope,))
|
||||
assert isinstance(reservation, BatchEnqueuedTokenReservation)
|
||||
await store.save_reservation("batch_local_record", reservation)
|
||||
assert not fake.records
|
||||
assert await raw.exists(_record("batch_local_record")) == 0
|
||||
|
||||
popped = await store.pop_reservation("batch_local_record")
|
||||
assert popped == reservation
|
||||
await store.refund(popped)
|
||||
assert not fake.counters
|
||||
assert await raw.exists(_counter(scope)) == 0
|
||||
assert await store.pop_reservation("batch_local_record") is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_local_ghost_left_by_landed_save_never_refunds_twice():
|
||||
scope = _scope(limit=100)
|
||||
record_key: Final = "batch_enqueued_token_reservation:batch_ghost"
|
||||
fake = _SingleKeyRedisFake(raise_after_landing_save_keys=frozenset({record_key}))
|
||||
store = BatchEnqueuedTokenStore(
|
||||
internal_usage_cache=InternalUsageCache(DualCache(redis_cache=fake, default_in_memory_ttl=60))
|
||||
)
|
||||
store, raw = _redis_store(fakeredis.FakeServer(), raise_after_landing_keys=frozenset({_record("batch_ghost")}))
|
||||
|
||||
reservation = await store.reserve(tokens=60, scopes=(scope,))
|
||||
assert isinstance(reservation, BatchEnqueuedTokenReservation)
|
||||
await store.save_reservation("batch_ghost", reservation)
|
||||
assert fake.records[record_key]
|
||||
assert await raw.get(_record("batch_ghost"))
|
||||
|
||||
first = await store.pop_reservation("batch_ghost")
|
||||
assert first == reservation
|
||||
await store.refund(first)
|
||||
assert not fake.counters
|
||||
assert fake.records[record_key] == ""
|
||||
assert await raw.exists(_counter(scope)) == 0
|
||||
assert await raw.get(_record("batch_ghost")) == b""
|
||||
|
||||
assert await store.pop_reservation("batch_ghost") is None
|
||||
assert (
|
||||
await store.internal_usage_cache.async_get_cache(
|
||||
key=record_key, litellm_parent_otel_span=None, local_only=True
|
||||
key=_record("batch_ghost"), litellm_parent_otel_span=None, local_only=True
|
||||
)
|
||||
is None
|
||||
)
|
||||
assert await store.pop_reservation("batch_ghost") is None
|
||||
assert isinstance(await store.reserve(tokens=100, scopes=(scope,)), BatchEnqueuedTokenReservation)
|
||||
assert fake.counters[f"batch_enqueued_tokens:{scope.key}:{scope.value}"] == 100
|
||||
assert await raw.get(_counter(scope)) == b"100"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_record_ttl_shrinks_by_elapsed_time_so_stale_records_never_outlive_their_counters():
|
||||
scope = _scope(limit=100)
|
||||
fake = _SingleKeyRedisFake()
|
||||
ticks = iter((1_000.0, 1_030.5))
|
||||
store = BatchEnqueuedTokenStore(
|
||||
internal_usage_cache=InternalUsageCache(DualCache(redis_cache=fake, default_in_memory_ttl=60)),
|
||||
monotonic=lambda: next(ticks),
|
||||
)
|
||||
store, raw = _redis_store(fakeredis.FakeServer(), monotonic=lambda: next(ticks))
|
||||
reservation = await store.reserve(tokens=60, scopes=(scope,))
|
||||
assert isinstance(reservation, BatchEnqueuedTokenReservation)
|
||||
assert reservation.reserved_at_monotonic == 1_000.0
|
||||
await store.save_reservation("batch_ttl_clamp", reservation)
|
||||
assert fake.save_ttls == (BATCH_ENQUEUED_TOKEN_TTL_SECONDS - 31,)
|
||||
assert await raw.ttl(_record("batch_ttl_clamp")) == BATCH_ENQUEUED_TOKEN_TTL_SECONDS - 31
|
||||
assert await store.pop_reservation("batch_ttl_clamp") == reservation
|
||||
|
||||
|
||||
|
|
@ -346,20 +348,15 @@ async def test_memory_refund_skips_reservations_granted_by_another_worker():
|
|||
@pytest.mark.asyncio
|
||||
async def test_failed_redis_refund_leaves_local_counters_untouched():
|
||||
scope = _scope(limit=100)
|
||||
counter_key: Final = f"batch_enqueued_tokens:api_key:{scope.value}"
|
||||
fake = _SingleKeyRedisFake(fail_refund_keys=frozenset({counter_key}))
|
||||
store = BatchEnqueuedTokenStore(
|
||||
internal_usage_cache=InternalUsageCache(DualCache(redis_cache=fake, default_in_memory_ttl=60))
|
||||
)
|
||||
store, raw = _redis_store(fakeredis.FakeServer(), fail_from_call=MappingProxyType({_counter(scope): 2}))
|
||||
|
||||
reservation = await store.reserve(tokens=60, scopes=(scope,))
|
||||
assert isinstance(reservation, BatchEnqueuedTokenReservation)
|
||||
assert reservation.backend == "redis"
|
||||
store.internal_usage_cache.dual_cache.in_memory_cache.set_cache(key=counter_key, value=45)
|
||||
store.internal_usage_cache.dual_cache.in_memory_cache.set_cache(key=_counter(scope), value=45)
|
||||
|
||||
await store.refund(reservation)
|
||||
assert store.internal_usage_cache.dual_cache.in_memory_cache.get_cache(key=counter_key) == 45
|
||||
assert fake.counters == {counter_key: 60}
|
||||
assert store.internal_usage_cache.dual_cache.in_memory_cache.get_cache(key=_counter(scope)) == 45
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -401,22 +398,18 @@ def test_batch_response_view_accepts_batch_objects_only():
|
|||
assert batch_response_view("batch_1") is None
|
||||
|
||||
|
||||
class _OpenBreakerRedis:
|
||||
def async_register_script(self, script: str):
|
||||
async def refused(keys: Sequence[str], args: Sequence[str | bytes | int | float]) -> object:
|
||||
raise RedisCircuitBreakerOpenError("Redis circuit breaker is open")
|
||||
|
||||
return refused
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_open_circuit_breaker_keeps_reservations_in_memory_without_a_warning(caplog):
|
||||
scope = _scope(limit=100)
|
||||
cache = fake_redis_cache(fakeredis.FakeServer())
|
||||
for _ in range(cache._circuit_breaker.failure_threshold):
|
||||
cache._circuit_breaker.record_failure()
|
||||
store = BatchEnqueuedTokenStore(
|
||||
internal_usage_cache=InternalUsageCache(DualCache(redis_cache=_OpenBreakerRedis(), default_in_memory_ttl=60)) # pyright: ignore[reportArgumentType] # duck-typed Redis double
|
||||
internal_usage_cache=InternalUsageCache(DualCache(redis_cache=cache, default_in_memory_ttl=60))
|
||||
)
|
||||
caplog.clear()
|
||||
|
||||
with caplog.at_level(logging.DEBUG, logger="LiteLLM Proxy"):
|
||||
with caplog.at_level(logging.DEBUG):
|
||||
reservation = await store.reserve(tokens=60, scopes=(scope,))
|
||||
assert isinstance(reservation, BatchEnqueuedTokenReservation)
|
||||
assert reservation.backend == "memory"
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ from typing import Any, Dict, Final, List, Optional
|
|||
from unittest.mock import patch
|
||||
|
||||
import httpx
|
||||
import fakeredis
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
from pydantic import TypeAdapter
|
||||
|
|
@ -7176,6 +7177,11 @@ async def test_post_call_success_hook_leaves_raw_provider_dict_untouched():
|
|||
|
||||
|
||||
class _OpenBreakerRedis:
|
||||
def transport(self):
|
||||
from tests._support.fake_redis import fake_redis_transport
|
||||
|
||||
return fake_redis_transport(fakeredis.FakeAsyncRedis(), failure_threshold=1)
|
||||
|
||||
def async_register_script(self, script: str):
|
||||
async def refused(keys, args):
|
||||
from litellm.caching.redis_cache import RedisCircuitBreakerOpenError
|
||||
|
|
@ -7224,6 +7230,11 @@ async def test_an_open_circuit_breaker_reads_the_sliding_window_locally_without_
|
|||
|
||||
|
||||
class _UnreachableRedis:
|
||||
def transport(self):
|
||||
from tests._support.fake_redis import fake_redis_transport
|
||||
|
||||
return fake_redis_transport(fakeredis.FakeAsyncRedis(), failure_threshold=1)
|
||||
|
||||
def async_register_script(self, script: str):
|
||||
async def refused(keys, args):
|
||||
raise ConnectionError("Error 61 connecting to 127.0.0.1:6379. Connection refused.")
|
||||
|
|
@ -7232,6 +7243,11 @@ class _UnreachableRedis:
|
|||
|
||||
|
||||
class _ScriptedRedis:
|
||||
def transport(self):
|
||||
from tests._support.fake_redis import fake_redis_transport
|
||||
|
||||
return fake_redis_transport(fakeredis.FakeAsyncRedis(), failure_threshold=1)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
failing_script: str | None = None,
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue