mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
Merge 1944a38928 into 3bd75f1258
This commit is contained in:
commit
e4aa7211c6
7 changed files with 877 additions and 21 deletions
|
|
@ -709,6 +709,11 @@ class RedisCache(BaseCache):
|
|||
def _admit_async_call(self, call: Callable[[], Awaitable[_RedisCallResult]]) -> Awaitable[_RedisCallResult]:
|
||||
return call()
|
||||
|
||||
@property
|
||||
def circuit_breaker_closed(self) -> bool:
|
||||
"""Whether calls reach Redis without a recovery probe. Reading it, unlike making a call, never starts one."""
|
||||
return not self._circuit_breaker.enabled or self._circuit_breaker.state == RedisCircuitBreaker.CLOSED
|
||||
|
||||
def _setup_health_pings(self):
|
||||
"""Setup async and sync health pings for Redis."""
|
||||
# ASYNC HEALTH PING
|
||||
|
|
|
|||
|
|
@ -1929,6 +1929,7 @@ SPEND_LOG_QUEUE_POLL_INTERVAL: Final = get_env_float("SPEND_LOG_QUEUE_POLL_INTER
|
|||
RESPONSES_SESSION_LOOKUP_MAX_ATTEMPTS: Final = max(1, get_env_int("RESPONSES_SESSION_LOOKUP_MAX_ATTEMPTS", 3))
|
||||
RESPONSES_SESSION_LOOKUP_RETRY_INTERVAL: Final = get_env_float("RESPONSES_SESSION_LOOKUP_RETRY_INTERVAL", 0.2)
|
||||
SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE: Final = get_env_int("SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE", 10000)
|
||||
SPEND_COUNTER_COLD_SEED_JOIN_SECONDS: Final = 0.05
|
||||
PROXY_DB_LOOKUP_MAX_CONCURRENCY: Final = max(1, get_env_int("PROXY_DB_LOOKUP_MAX_CONCURRENCY", 25))
|
||||
PROXY_DB_LOOKUP_DEADLINE_SECONDS: Final = max(0.1, get_env_float("PROXY_DB_LOOKUP_DEADLINE_SECONDS", 10.0))
|
||||
PROXY_DB_LOOKUP_STALL_WINDOW_SECONDS: Final = max(0.0, get_env_float("PROXY_DB_LOOKUP_STALL_WINDOW_SECONDS", 30.0))
|
||||
|
|
|
|||
|
|
@ -13,22 +13,27 @@ memory in long-lived deployments.
|
|||
"""
|
||||
|
||||
import asyncio
|
||||
import functools
|
||||
from collections import OrderedDict
|
||||
from collections.abc import Mapping
|
||||
from collections.abc import Awaitable, Callable, Mapping
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, ClassVar, Final, Optional
|
||||
|
||||
from litellm._internal_context import with_service_target
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.constants import SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE
|
||||
from litellm.constants import SPEND_COUNTER_COLD_SEED_JOIN_SECONDS, SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE
|
||||
from litellm.litellm_core_utils.duration_parser import duration_in_seconds
|
||||
from litellm.proxy._types import Litellm_EntityType
|
||||
from litellm.proxy.db.db_lookup_gate import bounded_db_lookup, db_lookup_gate
|
||||
from litellm.proxy.spend_tracking.spend_counter_batch import (
|
||||
SPEND_COUNTERS_TARGET,
|
||||
SpendCounterRead,
|
||||
forget_spend_counter,
|
||||
read_batched_spend_counter,
|
||||
record_spend_counter_value,
|
||||
spend_counter_tick,
|
||||
)
|
||||
from litellm.repositories.organization_repository import OrganizationRepository
|
||||
from litellm.repositories.project_repository import ProjectRepository
|
||||
|
|
@ -68,6 +73,42 @@ _WINDOW_SPEND_LOG_FIELDS: Final[Mapping[str, str]] = MappingProxyType(
|
|||
)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class _CachedSpendSeed:
|
||||
"""This process adding its cached spend to one cold counter: the add in flight, the tick it started at, and the
|
||||
tick the last one landed at."""
|
||||
|
||||
started_at: int = 0
|
||||
landed_at: int = 0
|
||||
in_flight: asyncio.Task[float | None] | None = None
|
||||
|
||||
def on_this_loop(self) -> bool:
|
||||
return self.in_flight is None or self.in_flight.get_loop() is asyncio.get_running_loop()
|
||||
|
||||
def start(self, add: Callable[[], Awaitable[float | None]]) -> asyncio.Task[float | None]:
|
||||
self.started_at = spend_counter_tick()
|
||||
in_flight: Final = asyncio.ensure_future(add())
|
||||
in_flight.add_done_callback(self._settle)
|
||||
self.in_flight = in_flight
|
||||
return in_flight
|
||||
|
||||
def _settle(self, done: asyncio.Future[float | None]) -> None:
|
||||
if self.in_flight is done:
|
||||
self.in_flight = None
|
||||
if not done.cancelled() and done.exception() is None:
|
||||
self.landed_at = spend_counter_tick()
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE)
|
||||
def _cached_spend_seed(spend_counter_cache: "DualCache", counter_key: str) -> _CachedSpendSeed:
|
||||
return _CachedSpendSeed()
|
||||
|
||||
|
||||
def _redis_breaker_closed(spend_counter_cache: "DualCache") -> bool:
|
||||
redis_cache: Final = spend_counter_cache.redis_cache
|
||||
return redis_cache is None or redis_cache.circuit_breaker_closed
|
||||
|
||||
|
||||
def _as_utc(value: datetime) -> datetime:
|
||||
return value if value.tzinfo is not None else value.replace(tzinfo=timezone.utc)
|
||||
|
||||
|
|
@ -112,6 +153,50 @@ class SpendCounterReseed:
|
|||
SpendCounterReseed._locks.popitem(last=False)
|
||||
return lock
|
||||
|
||||
@staticmethod
|
||||
async def add_cached_spend_once(
|
||||
spend_counter_cache: "DualCache",
|
||||
counter_key: str,
|
||||
read: SpendCounterRead,
|
||||
add: Callable[[], Awaitable[float | None]],
|
||||
is_warm: Callable[[], Awaitable[bool]],
|
||||
) -> None:
|
||||
"""Run ``add``, this process adding its cached spend to a cold counter, once for all of this process's
|
||||
requests that read the counter cold before that add landed. A request joins an add that started after its read
|
||||
came back and gets its result, and starts or joins another add when that one fails. When an add landed after
|
||||
its read went out, or one that started before its read came back finishes, the request re-reads the counter
|
||||
and adds again only if the counter is gone. A request waits on other requests' adds for at most
|
||||
``SPEND_COUNTER_COLD_SEED_JOIN_SECONDS`` in all and then adds its own. A request on another event loop than
|
||||
the add in flight, one that finds the Redis circuit breaker not closed, and each other process, still add their
|
||||
own cached spend."""
|
||||
seed: Final = _cached_spend_seed(spend_counter_cache, counter_key)
|
||||
loop: Final = asyncio.get_running_loop()
|
||||
join_by: Final = loop.time() + SPEND_COUNTER_COLD_SEED_JOIN_SECONDS
|
||||
|
||||
async def add_unless_covered(cold: SpendCounterRead) -> float | None:
|
||||
in_flight: Final = seed.in_flight
|
||||
if not seed.on_this_loop() or not _redis_breaker_closed(spend_counter_cache):
|
||||
return await add()
|
||||
if in_flight is None:
|
||||
if seed.landed_at < cold.sent_at:
|
||||
return await asyncio.shield(seed.start(add))
|
||||
forget_spend_counter(counter_key)
|
||||
sent_at: Final = spend_counter_tick()
|
||||
if await is_warm():
|
||||
return None
|
||||
return await add_unless_covered(SpendCounterRead(sent_at=sent_at, returned_at=spend_counter_tick()))
|
||||
covers: Final = seed.started_at > cold.returned_at
|
||||
await asyncio.wait((in_flight,), timeout=max(0.0, join_by - loop.time()))
|
||||
if not in_flight.done():
|
||||
return await asyncio.shield(seed.start(add))
|
||||
if covers and not in_flight.cancelled() and in_flight.exception() is None:
|
||||
return in_flight.result()
|
||||
return await add_unless_covered(cold)
|
||||
|
||||
added: Final = await add_unless_covered(read)
|
||||
if added is not None:
|
||||
record_spend_counter_value(counter_key, added)
|
||||
|
||||
@staticmethod
|
||||
@with_service_target(SPEND_COUNTERS_TARGET)
|
||||
async def increment_in_memory(spend_counter_cache: "DualCache", counter_key: str, increment: float) -> float | None:
|
||||
|
|
|
|||
|
|
@ -940,6 +940,8 @@ from litellm.proxy.spend_tracking.spend_counter_batch import (
|
|||
read_batched_spend_counter,
|
||||
record_spend_counter_value,
|
||||
spend_counter_batch_scope,
|
||||
spend_counter_read,
|
||||
spend_counter_tick,
|
||||
)
|
||||
from litellm.proxy.spend_tracking.spend_management_endpoints import (
|
||||
router as spend_management_router,
|
||||
|
|
@ -4143,7 +4145,9 @@ async def ensure_spend_counter_initialized(
|
|||
counter_key: str,
|
||||
source_cache_key: str | list[str],
|
||||
) -> None:
|
||||
sent_at: Final = spend_counter_tick()
|
||||
is_warm: Final = await _is_spend_counter_cache_warm(counter_key=counter_key)
|
||||
read: Final = spend_counter_read(counter_key, sent_at)
|
||||
if is_warm is False:
|
||||
# Shares the per-counter lock with get_current_spend.
|
||||
db_spend: Final = await SpendCounterReseed.coalesced(
|
||||
|
|
@ -4156,7 +4160,13 @@ async def ensure_spend_counter_initialized(
|
|||
# DB unavailable - fall back to in-process cache (may be stale).
|
||||
base_spend: Final = await _get_source_cache_base_spend(source_cache_key=source_cache_key)
|
||||
if base_spend > 0:
|
||||
await increment_spend_counter_cache(counter_key=counter_key, increment=base_spend)
|
||||
await SpendCounterReseed.add_cached_spend_once(
|
||||
spend_counter_cache=spend_counter_cache,
|
||||
counter_key=counter_key,
|
||||
read=read,
|
||||
add=lambda: increment_spend_counter_cache(counter_key=counter_key, increment=base_spend),
|
||||
is_warm=lambda: _is_spend_counter_cache_warm(counter_key=counter_key),
|
||||
)
|
||||
|
||||
|
||||
_ensure_spend_counter_initialized: Final = ensure_spend_counter_initialized
|
||||
|
|
|
|||
|
|
@ -1,11 +1,12 @@
|
|||
"""One Redis MGET per phase (admission, reservation, post-call) for the spend counters it reads, not one GET each."""
|
||||
|
||||
import asyncio
|
||||
import itertools
|
||||
from collections.abc import Iterator, Mapping, Sequence
|
||||
from contextvars import ContextVar, Token
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType, TracebackType
|
||||
from typing import Final
|
||||
from typing import Final, TypeAlias
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
|
|
@ -22,6 +23,24 @@ from litellm.proxy.common_utils.user_api_key_cache import (
|
|||
_CounterValues: Final = TypeAdapter(dict[str, float | None])
|
||||
_NO_VALUES: Final[Mapping[str, float | None]] = MappingProxyType({})
|
||||
SPEND_COUNTERS_TARGET: Final = "spend_counters"
|
||||
_ticks: Final = itertools.count(1)
|
||||
|
||||
|
||||
def spend_counter_tick() -> int:
|
||||
"""The next number in a per-process sequence that orders spend counter reads and cold seeds."""
|
||||
return next(_ticks)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SpendCounterRead:
|
||||
"""When a read of a spend counter went out and when its reply came back, as spend counter ticks."""
|
||||
|
||||
sent_at: int
|
||||
returned_at: int
|
||||
|
||||
|
||||
_NO_READS: Final[Mapping[str, SpendCounterRead]] = MappingProxyType({})
|
||||
_PendingMGet: TypeAlias = tuple[BatchResult[Mapping[str, object]], int, "asyncio.Future[int]"]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
@ -39,7 +58,17 @@ class SpendCounterBatch:
|
|||
hook declares whatever is bound but unread, so whoever flushes first (the auth object prefetch, usually)
|
||||
carries the spend counters in the same round trip."""
|
||||
|
||||
__slots__ = ("_fetched", "_inflight", "_keys", "_loaded", "_lock", "_open", "_redis_cache", "_request_batch")
|
||||
__slots__ = (
|
||||
"_fetched",
|
||||
"_inflight",
|
||||
"_keys",
|
||||
"_loaded",
|
||||
"_lock",
|
||||
"_open",
|
||||
"_reads",
|
||||
"_redis_cache",
|
||||
"_request_batch",
|
||||
)
|
||||
|
||||
def __init__(self, redis_cache: RedisCache) -> None:
|
||||
self._redis_cache: Final = redis_cache
|
||||
|
|
@ -48,7 +77,8 @@ class SpendCounterBatch:
|
|||
self._keys: frozenset[str] = frozenset()
|
||||
self._fetched: frozenset[str] = frozenset()
|
||||
self._loaded: Mapping[str, float | None] = _NO_VALUES
|
||||
self._inflight: Final[list[BatchResult[Mapping[str, object]]]] = [] # mutable-ok: drained by _load
|
||||
self._reads: Mapping[str, SpendCounterRead] = _NO_READS
|
||||
self._inflight: Final[list[_PendingMGet]] = [] # mutable-ok: drained by _load
|
||||
self._request_batch: Final[RedisBatch | None] = active_request_redis_batch(redis_cache)
|
||||
if self._request_batch is not None:
|
||||
self._request_batch.add_flush_hook(self._declare_pending)
|
||||
|
|
@ -78,6 +108,10 @@ class SpendCounterBatch:
|
|||
return None
|
||||
return loaded[counter_key], True
|
||||
|
||||
def counter_read(self, counter_key: str) -> SpendCounterRead | None:
|
||||
"""When the MGET behind this batch's value for the counter went out and came back, None without that value."""
|
||||
return self._reads.get(counter_key) if self._open and counter_key in self._loaded else None
|
||||
|
||||
def record(self, counter_key: str, value: float) -> None:
|
||||
"""A write returned the counter's new value; later reads in this scope see it instead of the MGET value."""
|
||||
if not self._open:
|
||||
|
|
@ -93,6 +127,7 @@ class SpendCounterBatch:
|
|||
self._keys = self._keys - key
|
||||
self._fetched = self._fetched - key
|
||||
self._loaded = MappingProxyType({k: v for k, v in self._loaded.items() if k != counter_key})
|
||||
self._reads = MappingProxyType({k: v for k, v in self._reads.items() if k != counter_key})
|
||||
|
||||
async def _load(self) -> Mapping[str, float | None]:
|
||||
async with self._lock:
|
||||
|
|
@ -103,10 +138,15 @@ class SpendCounterBatch:
|
|||
pending: Final = self._keys - self._fetched
|
||||
if pending:
|
||||
self._fetched = self._fetched | pending
|
||||
sent_at: Final = spend_counter_tick()
|
||||
fetched: Final = await self._fetch(pending)
|
||||
self._loaded = MappingProxyType({**fetched, **self._loaded})
|
||||
self._merge(fetched, SpendCounterRead(sent_at=sent_at, returned_at=spend_counter_tick()))
|
||||
return self._loaded
|
||||
|
||||
def _merge(self, fetched: Mapping[str, float | None], read: SpendCounterRead) -> None:
|
||||
self._loaded = MappingProxyType({**fetched, **self._loaded})
|
||||
self._reads = MappingProxyType({**dict.fromkeys(fetched, read), **self._reads})
|
||||
|
||||
def _declare_pending(self) -> None:
|
||||
"""Flush hook: put every bound-but-unread counter on the request pipeline that is about to go out."""
|
||||
if self._request_batch is None or not self._open:
|
||||
|
|
@ -114,19 +154,28 @@ class SpendCounterBatch:
|
|||
pending: Final = self._keys - self._fetched
|
||||
if pending:
|
||||
self._fetched = self._fetched | pending
|
||||
sent_at: Final = spend_counter_tick()
|
||||
with service_target(SPEND_COUNTERS_TARGET):
|
||||
self._inflight.append(self._request_batch.mget(sorted(pending)))
|
||||
read: Final = self._request_batch.mget(sorted(pending))
|
||||
returned_at: Final[asyncio.Future[int]] = asyncio.get_running_loop().create_future()
|
||||
read.on_settled(lambda _: returned_at.set_result(spend_counter_tick()))
|
||||
self._inflight.append((read, sent_at, returned_at))
|
||||
|
||||
async def _collect_inflight(self) -> None:
|
||||
results: Final = tuple(self._inflight)
|
||||
reads: Final = tuple(self._inflight)
|
||||
self._inflight.clear()
|
||||
for result in results:
|
||||
for read, sent_at, returned_at in reads:
|
||||
try:
|
||||
fetched: Mapping[str, float | None] = _CounterValues.validate_python(await result)
|
||||
fetched: Mapping[str, float | None] = _CounterValues.validate_python(await read)
|
||||
except Exception as e: # noqa: BLE001 # per-key reads take over and apply their own Redis fallback
|
||||
verbose_proxy_logger.debug("spend counter batch read failed, falling back to per-key reads: %s", e)
|
||||
continue
|
||||
self._loaded = MappingProxyType({**fetched, **self._loaded})
|
||||
self._merge(
|
||||
fetched,
|
||||
SpendCounterRead(
|
||||
sent_at=sent_at, returned_at=returned_at.result() if returned_at.done() else spend_counter_tick()
|
||||
),
|
||||
)
|
||||
|
||||
async def _fetch(self, keys: frozenset[str]) -> Mapping[str, float | None]:
|
||||
try:
|
||||
|
|
@ -145,6 +194,14 @@ def active_spend_counter_batch() -> SpendCounterBatch | None:
|
|||
return _active_batch.get()
|
||||
|
||||
|
||||
def spend_counter_read(counter_key: str, sent_at: int) -> SpendCounterRead:
|
||||
"""When this context's last read of the counter went out and came back: the open batch's MGET for it, else a read
|
||||
that went out at ``sent_at`` and came back now."""
|
||||
batch: Final = _active_batch.get()
|
||||
batched: Final = batch.counter_read(counter_key) if batch is not None else None
|
||||
return batched if batched is not None else SpendCounterRead(sent_at=sent_at, returned_at=spend_counter_tick())
|
||||
|
||||
|
||||
class spend_counter_batch_scope:
|
||||
"""Reads inside the scope share one MGET for the keys bound here or by ``bind_*`` calls inside it.
|
||||
Opened inside a scope whose batch is still open, it binds into that batch so both phases share the MGET."""
|
||||
|
|
|
|||
|
|
@ -1,17 +1,20 @@
|
|||
import json
|
||||
import os
|
||||
import re
|
||||
import uuid
|
||||
from collections.abc import Callable, Mapping
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from dataclasses import dataclass
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from integration._support.client import Gateway, Scenario, eventually, object_value, string_value
|
||||
from integration._support.database import read_rows
|
||||
from integration._support.database import read_rows, write_rows
|
||||
from integration._support.upstream import delete_scenario, register_scenario
|
||||
from integration.cost_calculation.cost_tracking_case import JsonResponse
|
||||
from pydantic import BaseModel, JsonValue
|
||||
from redis import Redis
|
||||
|
||||
PROVIDER_AUTHORIZATION: Final = "Bearer integration-provider-key"
|
||||
CANNED_CONTENT: Final = "Hello! This is a mock response from the fake OpenAI endpoint."
|
||||
|
|
@ -639,3 +642,141 @@ def test_streamed_call_is_refused_as_json_once_over_budget(gateway: Gateway) ->
|
|||
assert "data:" not in denied.text, denied.text
|
||||
_assert_budget_refusal(denied, end_user, 0.05)
|
||||
assert _observed_requests(upstream) == []
|
||||
|
||||
|
||||
_COLD_BURST: Final = 40
|
||||
_COLD_SEED_SPEND: Final = 6.0
|
||||
_COLD_BUDGET: Final = 10.0
|
||||
_CHAT_COST: Final = 20 * 0.001 + 20 * 0.002
|
||||
|
||||
|
||||
def _serves_model(gateway: Gateway, model: str) -> bool:
|
||||
response: Final = gateway.request("GET", "/model/info")
|
||||
return response.status_code == 200 and any(
|
||||
object_value(entry)["model_name"] == model for entry in object_value(response.json())["data"]
|
||||
)
|
||||
|
||||
|
||||
def _counter_cache() -> Redis:
|
||||
return Redis(host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"]))
|
||||
|
||||
|
||||
def _counter(cache: Redis, end_user: str) -> float | None:
|
||||
raw: Final = cache.get(f"spend:end_user:{end_user}")
|
||||
return None if raw is None else float(raw)
|
||||
|
||||
|
||||
def _lose_counter(cache: Redis, end_user: str) -> None:
|
||||
cache.delete(f"spend:end_user:{end_user}")
|
||||
|
||||
|
||||
def _set_end_user_row_spend(end_user: str, spend: float) -> None:
|
||||
write_rows('UPDATE "LiteLLM_EndUserTable" SET spend = %s WHERE user_id = %s', (str(spend), end_user))
|
||||
|
||||
|
||||
def _await_row_spend(end_user: str, spend: float) -> None:
|
||||
eventually(
|
||||
lambda: _end_user_row(end_user),
|
||||
lambda rows: len(rows) == 1 and float(str(rows[0]["spend"])) == pytest.approx(spend),
|
||||
seconds=70,
|
||||
)
|
||||
|
||||
|
||||
def _refused_spend(denied: httpx.Response, end_user: str) -> float:
|
||||
error: Final = _denial_error(denied)
|
||||
assert error["type"] == "budget_exceeded", denied.text
|
||||
matched: Final = _REFUSAL_MESSAGE.match(string_value(error["message"]))
|
||||
assert matched is not None, denied.text
|
||||
assert matched.group("user") == end_user
|
||||
return float(matched.group("spend"))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("spelling", ("user", "responses_user", "messages_metadata_user_id"))
|
||||
def test_overlapping_cold_starts_count_the_end_user_spend_once(gateway: Gateway, spelling: str) -> None:
|
||||
chosen: Final = SPELLINGS[spelling]
|
||||
with (
|
||||
gateway.scenario() as scenario,
|
||||
httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream,
|
||||
_counter_cache() as cache,
|
||||
):
|
||||
deployment: Final = _deployment(scenario, chosen)
|
||||
key: Final = scenario.key(models=[deployment.model])
|
||||
end_user: Final = f"integration-end-user-{uuid.uuid4().hex}"
|
||||
_new_customer(gateway, end_user, max_budget=_COLD_BUDGET, spend=_COLD_SEED_SPEND)
|
||||
scenario.cleanups.callback(gateway.request, "POST", "/customer/delete", {"user_ids": [end_user]})
|
||||
assert _customer_info(gateway, end_user).spend == _COLD_SEED_SPEND
|
||||
assert _counter(cache, end_user) is None
|
||||
upstream.delete("/__observations").raise_for_status()
|
||||
|
||||
with ThreadPoolExecutor(max_workers=_COLD_BURST) as pool:
|
||||
burst: Final = tuple(
|
||||
pool.map(
|
||||
lambda index: _send(gateway, chosen, deployment, key, end_user, f"cold {index}"),
|
||||
range(_COLD_BURST),
|
||||
)
|
||||
)
|
||||
refused: Final = [response.text for response in burst if response.status_code != 200]
|
||||
assert refused == [], refused
|
||||
follow_up: Final = _send(gateway, chosen, deployment, key, end_user, "cold follow-up")
|
||||
_assert_served(chosen, follow_up)
|
||||
assert len(_observed_requests(upstream)) == _COLD_BURST + 1
|
||||
|
||||
expected_spend: Final = _COLD_SEED_SPEND + (_COLD_BURST + 1) * _CHAT_COST
|
||||
_await_row_spend(end_user, expected_spend)
|
||||
assert _customer_info(gateway, end_user).spend == pytest.approx(expected_spend)
|
||||
counted: Final = eventually(
|
||||
lambda: _counter(cache, end_user),
|
||||
lambda value: value is not None and value == pytest.approx(expected_spend),
|
||||
seconds=30,
|
||||
return_last_on_timeout=True,
|
||||
)
|
||||
assert counted == pytest.approx(expected_spend)
|
||||
|
||||
|
||||
def test_a_lost_counter_keeps_spend_this_proxy_has_not_written_yet(gateway: Gateway) -> None:
|
||||
with gateway.scenario() as scenario, _counter_cache() as cache:
|
||||
model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.25)
|
||||
key: Final = scenario.key(models=[model])
|
||||
end_user: Final = f"integration-end-user-{uuid.uuid4().hex}"
|
||||
call_cost: Final = 20 * 0.001 + 20 * 0.25
|
||||
_new_customer(gateway, end_user, max_budget=_COLD_BUDGET, spend=_COLD_SEED_SPEND)
|
||||
scenario.cleanups.callback(gateway.request, "POST", "/customer/delete", {"user_ids": [end_user]})
|
||||
served: Final = _chat(gateway, model, key, end_user)
|
||||
assert served.status_code == 200, served.text
|
||||
_await_row_spend(end_user, _COLD_SEED_SPEND + call_cost)
|
||||
_set_end_user_row_spend(end_user, _COLD_SEED_SPEND)
|
||||
_lose_counter(cache, end_user)
|
||||
|
||||
denied: Final = _chat(gateway, model, key, end_user)
|
||||
|
||||
assert _refused_spend(denied, end_user) == pytest.approx(_COLD_SEED_SPEND + call_cost)
|
||||
|
||||
|
||||
def test_a_lost_counter_keeps_spend_another_proxy_has_not_written_yet(gateway: Gateway, peer: Gateway) -> None:
|
||||
with gateway.scenario() as scenario, _counter_cache() as cache:
|
||||
model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.149)
|
||||
eventually(lambda: _serves_model(peer, model), bool, seconds=30)
|
||||
key: Final = scenario.key(models=[model])
|
||||
end_user: Final = f"integration-end-user-{uuid.uuid4().hex}"
|
||||
call_cost: Final = 20 * 0.001 + 20 * 0.149
|
||||
_new_customer(gateway, end_user, max_budget=_COLD_BUDGET, spend=_COLD_SEED_SPEND)
|
||||
scenario.cleanups.callback(gateway.request, "POST", "/customer/delete", {"user_ids": [end_user]})
|
||||
first: Final = _chat(peer, model, key, end_user)
|
||||
assert first.status_code == 200, first.text
|
||||
_await_row_spend(end_user, _COLD_SEED_SPEND + call_cost)
|
||||
_set_end_user_row_spend(end_user, _COLD_SEED_SPEND)
|
||||
_lose_counter(cache, end_user)
|
||||
second: Final = _chat(peer, model, key, end_user)
|
||||
assert second.status_code == 200, second.text
|
||||
expected_spend: Final = _COLD_SEED_SPEND + 2 * call_cost
|
||||
counted: Final = eventually(
|
||||
lambda: _counter(cache, end_user),
|
||||
lambda value: value is not None and value == pytest.approx(expected_spend),
|
||||
seconds=30,
|
||||
return_last_on_timeout=True,
|
||||
)
|
||||
|
||||
denied: Final = _chat(gateway, model, key, end_user)
|
||||
|
||||
assert counted == pytest.approx(expected_spend)
|
||||
assert _denial_error(denied)["type"] == "budget_exceeded", denied.text
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ Pins covered:
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import threading
|
||||
from collections.abc import Awaitable, Callable
|
||||
from datetime import datetime
|
||||
from typing import Final, Literal, cast
|
||||
|
|
@ -36,10 +37,12 @@ from opentelemetry.trace import StatusCode
|
|||
|
||||
import litellm
|
||||
import litellm.proxy.proxy_server as ps
|
||||
from litellm._service_logger import ServiceLogging
|
||||
from litellm._internal_context import current_service_target
|
||||
from litellm._service_logger import ServiceLogging
|
||||
from litellm.caching.dual_cache import DualCache
|
||||
from litellm.caching.in_memory_cache import InMemoryCache
|
||||
from litellm.caching.redis_cache import RedisCache, RedisCircuitBreaker
|
||||
from litellm.caching.redis_batch import request_redis_batch_scope
|
||||
from litellm.caching.redis_cache import RedisCache, RedisCircuitBreaker, RedisCircuitBreakerOpenError
|
||||
from litellm.constants import GLOBAL_PROXY_SPEND_CACHE_KEY
|
||||
from litellm.integrations.opentelemetry import OpenTelemetry, OpenTelemetryConfig
|
||||
from litellm.models.end_user import LiteLLM_EndUserTable
|
||||
|
|
@ -50,6 +53,7 @@ from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import evict_loca
|
|||
from litellm.proxy.common_utils.cache_pydantic_utils import CacheCodec
|
||||
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache, end_user_cache_key, tag_cache_key
|
||||
from litellm.proxy.management_endpoints.team_endpoints import refresh_cached_team
|
||||
from litellm.proxy.spend_tracking.spend_counter_batch import active_spend_counter_batch, spend_counter_batch_scope
|
||||
|
||||
from .conftest import normalize
|
||||
|
||||
|
|
@ -1805,10 +1809,10 @@ class _FakeRedisRunningIncrementIfPresent(FakeAsyncRedis):
|
|||
|
||||
|
||||
class _RedisCacheOnClient(RedisCache):
|
||||
def __init__(self, client: FakeAsyncRedis) -> None:
|
||||
def __init__(self, client: FakeAsyncRedis, breaker: RedisCircuitBreaker | None = None) -> None:
|
||||
self._injected_client: Final = client
|
||||
self.namespace = None
|
||||
self._circuit_breaker = RedisCircuitBreaker(failure_threshold=5, recovery_timeout=30)
|
||||
self._circuit_breaker = breaker or RedisCircuitBreaker(failure_threshold=5, recovery_timeout=30)
|
||||
self.service_logger_obj = ServiceLogging()
|
||||
self.default_ttl = None
|
||||
|
||||
|
|
@ -1817,7 +1821,9 @@ class _RedisCacheOnClient(RedisCache):
|
|||
|
||||
|
||||
class _FakeRedisHoldingOneReply(FakeAsyncRedis):
|
||||
def __init__(self, server: FakeServer, held_increment: float, applied: asyncio.Event, released: asyncio.Event) -> None:
|
||||
def __init__(
|
||||
self, server: FakeServer, held_increment: float, applied: asyncio.Event, released: asyncio.Event
|
||||
) -> None:
|
||||
super().__init__(server=server)
|
||||
self._held_increment: Final = held_increment
|
||||
self._applied: Final = applied
|
||||
|
|
@ -1970,9 +1976,7 @@ async def test_update_cache_traces_the_global_spend_redis_call_under_the_request
|
|||
for span in exporter.get_finished_spans()
|
||||
if span.name == "redis" and span.attributes is not None and span.parent is not None
|
||||
)
|
||||
assert redis_spans == (
|
||||
("async_increment_if_present", expected_status, request_span.get_span_context().span_id),
|
||||
)
|
||||
assert redis_spans == (("async_increment_if_present", expected_status, request_span.get_span_context().span_id),)
|
||||
|
||||
|
||||
class _UnwritableKeyUserApiKeyCache(UserApiKeyCache):
|
||||
|
|
@ -2013,3 +2017,556 @@ async def test_update_cache_still_adds_spend_to_the_others_when_one_cached_objec
|
|||
1.25,
|
||||
10.25,
|
||||
)
|
||||
|
||||
|
||||
_COLD_SEED_SOURCE: Final = "end_user_id:customer-1"
|
||||
|
||||
|
||||
class _SharedRedis:
|
||||
"""The counter commands a cold start sends, yielding like a network call so overlapping requests interleave."""
|
||||
|
||||
circuit_breaker_closed: Final = True
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.values: Final[dict[str, float]] = {} # mutable-ok: stands in for the Redis keyspace
|
||||
self.failing = False
|
||||
|
||||
async def async_batch_get_cache(self, key_list: list[str], **_: object) -> dict[str, float | None]:
|
||||
await asyncio.sleep(0)
|
||||
return {key: self.values.get(key) for key in key_list}
|
||||
|
||||
async def async_get_cache(self, key: str, **_: object) -> float | None:
|
||||
await asyncio.sleep(0)
|
||||
return self.values.get(key)
|
||||
|
||||
async def async_increment(self, key: str, value: float, **_: object) -> float:
|
||||
await asyncio.sleep(0)
|
||||
if self.failing:
|
||||
raise ConnectionError("redis down")
|
||||
self.values[key] = self.values.get(key, 0.0) + value
|
||||
return self.values[key]
|
||||
|
||||
async def async_delete_cache(self, key: str) -> None:
|
||||
await asyncio.sleep(0)
|
||||
if not self.failing:
|
||||
self.values.pop(key, None)
|
||||
|
||||
|
||||
def _process(
|
||||
monkeypatch: pytest.MonkeyPatch, redis: _SharedRedis | RedisCache | None, cached_spend: float
|
||||
) -> DualCache:
|
||||
counters: Final = DualCache(in_memory_cache=InMemoryCache(), redis_cache=cast(RedisCache, redis))
|
||||
objects: Final = DualCache(in_memory_cache=InMemoryCache())
|
||||
objects.in_memory_cache.set_cache(key=_COLD_SEED_SOURCE, value={"spend": cached_spend})
|
||||
monkeypatch.setattr(ps, "spend_counter_cache", counters)
|
||||
monkeypatch.setattr(ps, "user_api_key_cache", objects)
|
||||
monkeypatch.setattr(ps, "prisma_client", object())
|
||||
return counters
|
||||
|
||||
|
||||
async def _cold_start(redis: _SharedRedis | RedisCache | None, counter: str) -> None:
|
||||
with spend_counter_batch_scope(cast(RedisCache, redis) if redis is not None else None, frozenset((counter,))):
|
||||
await ps.ensure_spend_counter_initialized(counter_key=counter, source_cache_key=_COLD_SEED_SOURCE)
|
||||
|
||||
|
||||
async def _read_cold(counter: str) -> None:
|
||||
batch: Final = active_spend_counter_batch()
|
||||
assert batch is not None
|
||||
assert await batch.read(counter) == (None, True)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("with_redis", (True, False))
|
||||
async def test_overlapping_cold_starts_in_one_process_add_its_cached_spend_once(
|
||||
monkeypatch: pytest.MonkeyPatch, with_redis: bool
|
||||
) -> None:
|
||||
counter: Final = f"spend:end_user:once-{with_redis}"
|
||||
redis: Final = _SharedRedis() if with_redis else None
|
||||
counters: Final = _process(monkeypatch, redis, cached_spend=6.0)
|
||||
|
||||
await asyncio.gather(*(_cold_start(redis, counter) for _ in range(5)))
|
||||
|
||||
seeded: Final = redis.values.get(counter) if redis is not None else counters.in_memory_cache.get_cache(key=counter)
|
||||
assert seeded == 6.0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_request_that_read_the_counter_before_this_process_seeded_it_does_not_add_again(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
counter: Final = "spend:end_user:stale-reader"
|
||||
redis: Final = _SharedRedis()
|
||||
_process(monkeypatch, redis, cached_spend=6.0)
|
||||
read: Final = asyncio.Event()
|
||||
|
||||
async def _stale_request() -> None:
|
||||
with spend_counter_batch_scope(cast(RedisCache, redis), frozenset((counter,))):
|
||||
await _read_cold(counter)
|
||||
read.set()
|
||||
await seeded.wait()
|
||||
await ps.ensure_spend_counter_initialized(counter_key=counter, source_cache_key=_COLD_SEED_SOURCE)
|
||||
|
||||
seeded: Final = asyncio.Event()
|
||||
|
||||
async def _first_request() -> None:
|
||||
await read.wait()
|
||||
await _cold_start(redis, counter)
|
||||
seeded.set()
|
||||
|
||||
await asyncio.gather(_stale_request(), _first_request())
|
||||
|
||||
assert redis.values[counter] == 6.0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_counter_lost_again_after_this_process_seeded_it_is_seeded_again(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
counter: Final = "spend:end_user:lost-again"
|
||||
redis: Final = _SharedRedis()
|
||||
_process(monkeypatch, redis, cached_spend=6.0)
|
||||
read: Final = asyncio.Event()
|
||||
seeded: Final = asyncio.Event()
|
||||
|
||||
async def _stale_request() -> None:
|
||||
with spend_counter_batch_scope(cast(RedisCache, redis), frozenset((counter,))):
|
||||
await _read_cold(counter)
|
||||
read.set()
|
||||
await seeded.wait()
|
||||
del redis.values[counter]
|
||||
await ps.ensure_spend_counter_initialized(counter_key=counter, source_cache_key=_COLD_SEED_SOURCE)
|
||||
|
||||
async def _first_request() -> None:
|
||||
await read.wait()
|
||||
await _cold_start(redis, counter)
|
||||
seeded.set()
|
||||
|
||||
await asyncio.gather(_stale_request(), _first_request())
|
||||
|
||||
assert redis.values[counter] == 6.0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_another_process_seeding_after_this_request_read_the_counter_still_sums(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
counter: Final = "spend:end_user:two-processes"
|
||||
redis: Final = _SharedRedis()
|
||||
read: Final = asyncio.Event()
|
||||
a_seeded: Final = asyncio.Event()
|
||||
|
||||
async def _process_b() -> None:
|
||||
with spend_counter_batch_scope(cast(RedisCache, redis), frozenset((counter,))):
|
||||
await _read_cold(counter)
|
||||
read.set()
|
||||
await a_seeded.wait()
|
||||
_process(monkeypatch, redis, cached_spend=5.0)
|
||||
await ps.ensure_spend_counter_initialized(counter_key=counter, source_cache_key=_COLD_SEED_SOURCE)
|
||||
|
||||
async def _process_a() -> None:
|
||||
await read.wait()
|
||||
_process(monkeypatch, redis, cached_spend=3.0)
|
||||
await _cold_start(redis, counter)
|
||||
a_seeded.set()
|
||||
|
||||
await asyncio.gather(_process_b(), _process_a())
|
||||
|
||||
assert redis.values[counter] == 8.0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_seeds_that_keep_failing_fail_every_cold_start_and_the_next_cold_start_seeds(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
counter: Final = "spend:end_user:failed-seed"
|
||||
redis: Final = _SharedRedis()
|
||||
_process(monkeypatch, redis, cached_spend=6.0)
|
||||
redis.failing = True
|
||||
|
||||
outcomes: Final = await asyncio.gather(*(_cold_start(redis, counter) for _ in range(3)), return_exceptions=True)
|
||||
redis.failing = False
|
||||
await _cold_start(redis, counter)
|
||||
|
||||
assert [type(outcome) for outcome in outcomes] == [ConnectionError] * 3
|
||||
assert redis.values[counter] == 6.0
|
||||
|
||||
|
||||
class _HeldReplyRedis(_SharedRedis):
|
||||
"""Applies an increment at once but holds its reply until released, like a slow network round trip."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.applied: Final = asyncio.Event()
|
||||
self.release: Final = asyncio.Event()
|
||||
|
||||
async def async_increment(self, key: str, value: float, **_: object) -> float:
|
||||
self.values[key] = self.values.get(key, 0.0) + value
|
||||
total: Final = self.values[key]
|
||||
self.applied.set()
|
||||
await self.release.wait()
|
||||
return total
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_counter_deleted_before_an_earlier_add_returns_is_seeded_by_a_later_cold_reader(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
counter: Final = "spend:end_user:deleted-mid-add"
|
||||
redis: Final = _HeldReplyRedis()
|
||||
_process(monkeypatch, redis, cached_spend=6.0)
|
||||
|
||||
first: Final = asyncio.ensure_future(_cold_start(redis, counter))
|
||||
await redis.applied.wait()
|
||||
del redis.values[counter]
|
||||
later: Final = asyncio.ensure_future(_cold_start(redis, counter))
|
||||
for _ in range(20):
|
||||
await asyncio.sleep(0)
|
||||
redis.release.set()
|
||||
await asyncio.gather(first, later)
|
||||
|
||||
assert redis.values[counter] == 6.0
|
||||
|
||||
|
||||
def test_a_cold_start_on_another_event_loop_adds_on_its_own(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
counter: Final = "spend:end_user:two-loops"
|
||||
held: Final = threading.Event()
|
||||
release: Final = threading.Event()
|
||||
|
||||
class _ThreadHeldRedis(_SharedRedis):
|
||||
async def async_increment(self, key: str, value: float, **_: object) -> float:
|
||||
if threading.current_thread() is not threading.main_thread():
|
||||
held.set()
|
||||
while not release.is_set():
|
||||
await asyncio.sleep(0.001)
|
||||
return await super().async_increment(key, value)
|
||||
|
||||
redis: Final = _ThreadHeldRedis()
|
||||
_process(monkeypatch, redis, cached_spend=6.0)
|
||||
other_loop: Final = threading.Thread(target=lambda: asyncio.run(_cold_start(redis, counter)), daemon=True)
|
||||
other_loop.start()
|
||||
assert held.wait(5)
|
||||
|
||||
try:
|
||||
asyncio.run(_cold_start(redis, counter))
|
||||
finally:
|
||||
release.set()
|
||||
other_loop.join(5)
|
||||
|
||||
assert redis.values[counter] == 12.0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_request_that_read_the_counter_cold_after_this_process_seeded_it_still_adds_its_cached_spend(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
counter: Final = "spend:end_user:read-after-seed"
|
||||
redis: Final = _SharedRedis()
|
||||
_process(monkeypatch, redis, cached_spend=6.0)
|
||||
opened: Final = asyncio.Event()
|
||||
seeded: Final = asyncio.Event()
|
||||
|
||||
async def _late_reader() -> None:
|
||||
with spend_counter_batch_scope(cast(RedisCache, redis), frozenset((counter,))):
|
||||
opened.set()
|
||||
await seeded.wait()
|
||||
del redis.values[counter]
|
||||
await _read_cold(counter)
|
||||
redis.values[counter] = 0.5
|
||||
await ps.ensure_spend_counter_initialized(counter_key=counter, source_cache_key=_COLD_SEED_SOURCE)
|
||||
|
||||
async def _first_request() -> None:
|
||||
await opened.wait()
|
||||
await _cold_start(redis, counter)
|
||||
seeded.set()
|
||||
|
||||
await asyncio.gather(_late_reader(), _first_request())
|
||||
|
||||
assert redis.values[counter] == 6.5
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_request_whose_pipelined_read_returned_before_this_process_seeded_the_counter_does_not_add_again(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
counter: Final = "spend:end_user:pipelined-reader"
|
||||
client: Final = FakeAsyncRedis(server=FakeServer())
|
||||
redis: Final = _RedisCacheOnClient(client)
|
||||
_process(monkeypatch, redis, cached_spend=6.0)
|
||||
read: Final = asyncio.Event()
|
||||
seeded: Final = asyncio.Event()
|
||||
|
||||
async def _pipelined_request() -> None:
|
||||
with request_redis_batch_scope() as batches, spend_counter_batch_scope(redis, frozenset((counter,))):
|
||||
await batches.batch(redis).flush()
|
||||
read.set()
|
||||
await seeded.wait()
|
||||
await ps.ensure_spend_counter_initialized(counter_key=counter, source_cache_key=_COLD_SEED_SOURCE)
|
||||
|
||||
async def _first_request() -> None:
|
||||
await read.wait()
|
||||
await _cold_start(redis, counter)
|
||||
seeded.set()
|
||||
|
||||
await asyncio.gather(_pipelined_request(), _first_request())
|
||||
|
||||
assert float(await client.get(counter) or 0) == 6.0
|
||||
|
||||
|
||||
class _DroppedFirstIncrementRedis(_SharedRedis):
|
||||
"""Resets the connection that carries the first increment and answers every later one."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.dropped = False
|
||||
|
||||
async def async_increment(self, key: str, value: float, **_: object) -> float:
|
||||
if not self.dropped:
|
||||
self.dropped = True
|
||||
await asyncio.sleep(0)
|
||||
raise ConnectionError("connection reset by peer")
|
||||
return await super().async_increment(key, value)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_requests_that_joined_a_seed_whose_connection_reset_seed_again_and_only_its_starter_fails(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
counter: Final = "spend:end_user:reset-seed"
|
||||
redis: Final = _DroppedFirstIncrementRedis()
|
||||
_process(monkeypatch, redis, cached_spend=6.0)
|
||||
|
||||
outcomes: Final = await asyncio.gather(*(_cold_start(redis, counter) for _ in range(4)), return_exceptions=True)
|
||||
|
||||
failures: Final = [outcome for outcome in outcomes if outcome is not None]
|
||||
assert [type(failure) for failure in failures] == [ConnectionError], outcomes
|
||||
assert redis.values[counter] == 6.0
|
||||
|
||||
|
||||
class _StalledFirstIncrementRedis(_SharedRedis):
|
||||
"""Never answers the first increment until released, like a connection that stopped responding."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.stalled: Final = asyncio.Event()
|
||||
self.release: Final = asyncio.Event()
|
||||
|
||||
async def async_increment(self, key: str, value: float, **_: object) -> float:
|
||||
if not self.stalled.is_set():
|
||||
self.stalled.set()
|
||||
await self.release.wait()
|
||||
raise TimeoutError("redis did not answer")
|
||||
return await super().async_increment(key, value)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cold_starts_stop_waiting_on_a_stalled_seed_and_add_their_own(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
counter: Final = "spend:end_user:stalled-seed"
|
||||
redis: Final = _StalledFirstIncrementRedis()
|
||||
_process(monkeypatch, redis, cached_spend=6.0)
|
||||
stalled: Final = asyncio.ensure_future(_cold_start(redis, counter))
|
||||
await redis.stalled.wait()
|
||||
|
||||
try:
|
||||
await asyncio.wait_for(asyncio.gather(*(_cold_start(redis, counter) for _ in range(3))), timeout=10)
|
||||
seeded_while_stalled: Final = redis.values.get(counter)
|
||||
finally:
|
||||
redis.release.set()
|
||||
|
||||
with pytest.raises(TimeoutError):
|
||||
await stalled
|
||||
assert seeded_while_stalled == 18.0
|
||||
|
||||
|
||||
class _FakeRedisHoldingIncrements(FakeAsyncRedis):
|
||||
def __init__(self, server: FakeServer) -> None:
|
||||
super().__init__(server=server)
|
||||
self.holding: Final = asyncio.Event()
|
||||
self.release: Final = asyncio.Event()
|
||||
|
||||
async def incrbyfloat(self, name: str, amount: float = 1.0) -> float:
|
||||
self.holding.set()
|
||||
await self.release.wait()
|
||||
return await super().incrbyfloat(name, amount)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_cold_start_after_the_redis_breaker_opened_fails_fast_instead_of_using_an_add_in_flight(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
counter: Final = "spend:end_user:breaker-open"
|
||||
breaker: Final = RedisCircuitBreaker(failure_threshold=1, recovery_timeout=30)
|
||||
client: Final = _FakeRedisHoldingIncrements(FakeServer())
|
||||
redis: Final = _RedisCacheOnClient(client, breaker)
|
||||
_process(monkeypatch, redis, cached_spend=6.0)
|
||||
first: Final = asyncio.ensure_future(_cold_start(redis, counter))
|
||||
await client.holding.wait()
|
||||
breaker.record_failure()
|
||||
|
||||
later: Final = asyncio.ensure_future(_cold_start(redis, counter))
|
||||
for _ in range(100):
|
||||
await asyncio.sleep(0)
|
||||
client.release.set()
|
||||
|
||||
outcomes: Final = await asyncio.gather(first, later, return_exceptions=True)
|
||||
|
||||
assert outcomes[0] is None and isinstance(outcomes[1], RedisCircuitBreakerOpenError), outcomes
|
||||
|
||||
|
||||
class _HeldFirstMGetRedis(_SharedRedis):
|
||||
"""Reads the first MGET's keys at once but holds its reply until released, like a slow network round trip."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.mget_read: Final = asyncio.Event()
|
||||
self.release_mget: Final = asyncio.Event()
|
||||
|
||||
async def async_batch_get_cache(self, key_list: list[str], **_: object) -> dict[str, float | None]:
|
||||
values: Final = {key: self.values.get(key) for key in key_list}
|
||||
if not self.mget_read.is_set():
|
||||
self.mget_read.set()
|
||||
await self.release_mget.wait()
|
||||
return values
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_seed_that_lands_while_a_cold_read_is_on_its_way_back_is_not_added_again(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
counter: Final = "spend:end_user:seed-during-read"
|
||||
redis: Final = _HeldFirstMGetRedis()
|
||||
_process(monkeypatch, redis, cached_spend=6.0)
|
||||
|
||||
async def _first_request() -> None:
|
||||
await redis.mget_read.wait()
|
||||
await _cold_start(redis, counter)
|
||||
redis.release_mget.set()
|
||||
|
||||
await asyncio.gather(_cold_start(redis, counter), _first_request())
|
||||
|
||||
assert redis.values[counter] == 6.0
|
||||
|
||||
|
||||
class _HeldReadRedis(_SharedRedis):
|
||||
"""Answers an MGET with the values its keys had when it arrived, holding the reply while ``hold_reads`` is set."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.hold_reads = False
|
||||
self.read_arrived: Final = asyncio.Event()
|
||||
self.release_read: Final = asyncio.Event()
|
||||
|
||||
async def async_batch_get_cache(self, key_list: list[str], **_: object) -> dict[str, float | None]:
|
||||
values: Final = {key: self.values.get(key) for key in key_list}
|
||||
if self.hold_reads:
|
||||
self.read_arrived.set()
|
||||
await self.release_read.wait()
|
||||
return values
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_seed_that_lands_while_a_request_re_reads_the_counter_is_not_added_again(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
counter: Final = "spend:end_user:seed-during-reread"
|
||||
redis: Final = _HeldReadRedis()
|
||||
_process(monkeypatch, redis, cached_spend=6.0)
|
||||
read: Final = asyncio.Event()
|
||||
seeded: Final = asyncio.Event()
|
||||
|
||||
async def _stale_request() -> None:
|
||||
with spend_counter_batch_scope(cast(RedisCache, redis), frozenset((counter,))):
|
||||
await _read_cold(counter)
|
||||
read.set()
|
||||
await seeded.wait()
|
||||
del redis.values[counter]
|
||||
redis.hold_reads = True
|
||||
await ps.ensure_spend_counter_initialized(counter_key=counter, source_cache_key=_COLD_SEED_SOURCE)
|
||||
|
||||
async def _first_request() -> None:
|
||||
await read.wait()
|
||||
await _cold_start(redis, counter)
|
||||
seeded.set()
|
||||
|
||||
async def _request_seeding_during_the_re_read() -> None:
|
||||
await redis.read_arrived.wait()
|
||||
redis.hold_reads = False
|
||||
await _cold_start(redis, counter)
|
||||
redis.release_read.set()
|
||||
|
||||
await asyncio.gather(_stale_request(), _first_request(), _request_seeding_during_the_re_read())
|
||||
|
||||
assert redis.values[counter] == 6.0
|
||||
|
||||
|
||||
class _HeldGetRedis(_SharedRedis):
|
||||
"""Answers the first GET with the value its key had when it arrived, holding the reply until released."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.get_arrived: Final = asyncio.Event()
|
||||
self.release_get: Final = asyncio.Event()
|
||||
|
||||
async def async_get_cache(self, key: str, **_: object) -> float | None:
|
||||
value: Final = self.values.get(key)
|
||||
if not self.get_arrived.is_set():
|
||||
self.get_arrived.set()
|
||||
await self.release_get.wait()
|
||||
return value
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_seed_that_lands_while_an_unbatched_cold_read_is_on_its_way_back_is_not_added_again(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
counter: Final = "spend:end_user:seed-during-unbatched-read"
|
||||
redis: Final = _HeldGetRedis()
|
||||
_process(monkeypatch, redis, cached_spend=6.0)
|
||||
|
||||
async def _first_request() -> None:
|
||||
await redis.get_arrived.wait()
|
||||
await _cold_start(redis, counter)
|
||||
redis.release_get.set()
|
||||
|
||||
await asyncio.gather(
|
||||
ps.ensure_spend_counter_initialized(counter_key=counter, source_cache_key=_COLD_SEED_SOURCE),
|
||||
_first_request(),
|
||||
)
|
||||
|
||||
assert redis.values[counter] == 6.0
|
||||
|
||||
|
||||
class _QueuedReadRedis(_HeldReplyRedis):
|
||||
"""Holds the first MGET before Redis runs it, while increments apply at once and hold their replies."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.read_sent: Final = asyncio.Event()
|
||||
self.run_read: Final = asyncio.Event()
|
||||
|
||||
async def async_batch_get_cache(self, key_list: list[str], **_: object) -> dict[str, float | None]:
|
||||
if not self.read_sent.is_set():
|
||||
self.read_sent.set()
|
||||
await self.run_read.wait()
|
||||
return {key: self.values.get(key) for key in key_list}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_cold_read_sent_before_an_add_started_does_not_join_it_when_the_counter_was_deleted_after_it_landed(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
counter: Final = "spend:end_user:deleted-while-read-queued"
|
||||
redis: Final = _QueuedReadRedis()
|
||||
_process(monkeypatch, redis, cached_spend=6.0)
|
||||
|
||||
reader: Final = asyncio.ensure_future(_cold_start(redis, counter))
|
||||
await redis.read_sent.wait()
|
||||
first: Final = asyncio.ensure_future(_cold_start(redis, counter))
|
||||
await redis.applied.wait()
|
||||
del redis.values[counter]
|
||||
redis.run_read.set()
|
||||
for _ in range(20):
|
||||
await asyncio.sleep(0)
|
||||
redis.release.set()
|
||||
await asyncio.gather(reader, first)
|
||||
|
||||
assert redis.values.get(counter) == 6.0
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue