This commit is contained in:
yucheng-berri 2026-10-31 22:45:19 +00:00 • committed by GitHub
commit e4aa7211c6
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 877 additions and 21 deletions

View file

@ -709,6 +709,11 @@ class RedisCache(BaseCache):
def _admit_async_call(self, call: Callable[[], Awaitable[_RedisCallResult]]) -> Awaitable[_RedisCallResult]:
return call()
@property
def circuit_breaker_closed(self) -> bool:
"""Whether calls reach Redis without a recovery probe. Reading it, unlike making a call, never starts one."""
return not self._circuit_breaker.enabled or self._circuit_breaker.state == RedisCircuitBreaker.CLOSED
def _setup_health_pings(self):
"""Setup async and sync health pings for Redis."""
# ASYNC HEALTH PING

View file

@ -1929,6 +1929,7 @@ SPEND_LOG_QUEUE_POLL_INTERVAL: Final = get_env_float("SPEND_LOG_QUEUE_POLL_INTER
RESPONSES_SESSION_LOOKUP_MAX_ATTEMPTS: Final = max(1, get_env_int("RESPONSES_SESSION_LOOKUP_MAX_ATTEMPTS", 3))
RESPONSES_SESSION_LOOKUP_RETRY_INTERVAL: Final = get_env_float("RESPONSES_SESSION_LOOKUP_RETRY_INTERVAL", 0.2)
SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE: Final = get_env_int("SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE", 10000)
SPEND_COUNTER_COLD_SEED_JOIN_SECONDS: Final = 0.05
PROXY_DB_LOOKUP_MAX_CONCURRENCY: Final = max(1, get_env_int("PROXY_DB_LOOKUP_MAX_CONCURRENCY", 25))
PROXY_DB_LOOKUP_DEADLINE_SECONDS: Final = max(0.1, get_env_float("PROXY_DB_LOOKUP_DEADLINE_SECONDS", 10.0))
PROXY_DB_LOOKUP_STALL_WINDOW_SECONDS: Final = max(0.0, get_env_float("PROXY_DB_LOOKUP_STALL_WINDOW_SECONDS", 30.0))

View file

@ -13,22 +13,27 @@ memory in long-lived deployments.
"""
import asyncio
import functools
from collections import OrderedDict
from collections.abc import Mapping
from collections.abc import Awaitable, Callable, Mapping
from dataclasses import dataclass
from datetime import datetime, timezone
from types import MappingProxyType
from typing import TYPE_CHECKING, ClassVar, Final, Optional
from litellm._internal_context import with_service_target
from litellm._logging import verbose_proxy_logger
from litellm.constants import SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE
from litellm.constants import SPEND_COUNTER_COLD_SEED_JOIN_SECONDS, SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE
from litellm.litellm_core_utils.duration_parser import duration_in_seconds
from litellm.proxy._types import Litellm_EntityType
from litellm.proxy.db.db_lookup_gate import bounded_db_lookup, db_lookup_gate
from litellm.proxy.spend_tracking.spend_counter_batch import (
SPEND_COUNTERS_TARGET,
SpendCounterRead,
forget_spend_counter,
read_batched_spend_counter,
record_spend_counter_value,
spend_counter_tick,
)
from litellm.repositories.organization_repository import OrganizationRepository
from litellm.repositories.project_repository import ProjectRepository
@ -68,6 +73,42 @@ _WINDOW_SPEND_LOG_FIELDS: Final[Mapping[str, str]] = MappingProxyType(
)
@dataclass(slots=True)
class _CachedSpendSeed:
"""This process adding its cached spend to one cold counter: the add in flight, the tick it started at, and the
tick the last one landed at."""
started_at: int = 0
landed_at: int = 0
in_flight: asyncio.Task[float | None] | None = None
def on_this_loop(self) -> bool:
return self.in_flight is None or self.in_flight.get_loop() is asyncio.get_running_loop()
def start(self, add: Callable[[], Awaitable[float | None]]) -> asyncio.Task[float | None]:
self.started_at = spend_counter_tick()
in_flight: Final = asyncio.ensure_future(add())
in_flight.add_done_callback(self._settle)
self.in_flight = in_flight
return in_flight
def _settle(self, done: asyncio.Future[float | None]) -> None:
if self.in_flight is done:
self.in_flight = None
if not done.cancelled() and done.exception() is None:
self.landed_at = spend_counter_tick()
@functools.lru_cache(maxsize=SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE)
def _cached_spend_seed(spend_counter_cache: "DualCache", counter_key: str) -> _CachedSpendSeed:
return _CachedSpendSeed()
def _redis_breaker_closed(spend_counter_cache: "DualCache") -> bool:
redis_cache: Final = spend_counter_cache.redis_cache
return redis_cache is None or redis_cache.circuit_breaker_closed
def _as_utc(value: datetime) -> datetime:
return value if value.tzinfo is not None else value.replace(tzinfo=timezone.utc)
@ -112,6 +153,50 @@ class SpendCounterReseed:
SpendCounterReseed._locks.popitem(last=False)
return lock
@staticmethod
async def add_cached_spend_once(
spend_counter_cache: "DualCache",
counter_key: str,
read: SpendCounterRead,
add: Callable[[], Awaitable[float | None]],
is_warm: Callable[[], Awaitable[bool]],
) -> None:
"""Run ``add``, this process adding its cached spend to a cold counter, once for all of this process's
requests that read the counter cold before that add landed. A request joins an add that started after its read
came back and gets its result, and starts or joins another add when that one fails. When an add landed after
its read went out, or one that started before its read came back finishes, the request re-reads the counter
and adds again only if the counter is gone. A request waits on other requests' adds for at most
``SPEND_COUNTER_COLD_SEED_JOIN_SECONDS`` in all and then adds its own. A request on another event loop than
the add in flight, one that finds the Redis circuit breaker not closed, and each other process, still add their
own cached spend."""
seed: Final = _cached_spend_seed(spend_counter_cache, counter_key)
loop: Final = asyncio.get_running_loop()
join_by: Final = loop.time() + SPEND_COUNTER_COLD_SEED_JOIN_SECONDS
async def add_unless_covered(cold: SpendCounterRead) -> float | None:
in_flight: Final = seed.in_flight
if not seed.on_this_loop() or not _redis_breaker_closed(spend_counter_cache):
return await add()
if in_flight is None:
if seed.landed_at < cold.sent_at:
return await asyncio.shield(seed.start(add))
forget_spend_counter(counter_key)
sent_at: Final = spend_counter_tick()
if await is_warm():
return None
return await add_unless_covered(SpendCounterRead(sent_at=sent_at, returned_at=spend_counter_tick()))
covers: Final = seed.started_at > cold.returned_at
await asyncio.wait((in_flight,), timeout=max(0.0, join_by - loop.time()))
if not in_flight.done():
return await asyncio.shield(seed.start(add))
if covers and not in_flight.cancelled() and in_flight.exception() is None:
return in_flight.result()
return await add_unless_covered(cold)
added: Final = await add_unless_covered(read)
if added is not None:
record_spend_counter_value(counter_key, added)
@staticmethod
@with_service_target(SPEND_COUNTERS_TARGET)
async def increment_in_memory(spend_counter_cache: "DualCache", counter_key: str, increment: float) -> float | None:

View file

@ -940,6 +940,8 @@ from litellm.proxy.spend_tracking.spend_counter_batch import (
read_batched_spend_counter,
record_spend_counter_value,
spend_counter_batch_scope,
spend_counter_read,
spend_counter_tick,
)
from litellm.proxy.spend_tracking.spend_management_endpoints import (
router as spend_management_router,
@ -4143,7 +4145,9 @@ async def ensure_spend_counter_initialized(
counter_key: str,
source_cache_key: str | list[str],
) -> None:
sent_at: Final = spend_counter_tick()
is_warm: Final = await _is_spend_counter_cache_warm(counter_key=counter_key)
read: Final = spend_counter_read(counter_key, sent_at)
if is_warm is False:
# Shares the per-counter lock with get_current_spend.
db_spend: Final = await SpendCounterReseed.coalesced(
@ -4156,7 +4160,13 @@ async def ensure_spend_counter_initialized(
# DB unavailable - fall back to in-process cache (may be stale).
base_spend: Final = await _get_source_cache_base_spend(source_cache_key=source_cache_key)
if base_spend > 0:
await increment_spend_counter_cache(counter_key=counter_key, increment=base_spend)
await SpendCounterReseed.add_cached_spend_once(
spend_counter_cache=spend_counter_cache,
counter_key=counter_key,
read=read,
add=lambda: increment_spend_counter_cache(counter_key=counter_key, increment=base_spend),
is_warm=lambda: _is_spend_counter_cache_warm(counter_key=counter_key),
)
_ensure_spend_counter_initialized: Final = ensure_spend_counter_initialized

View file

@ -1,11 +1,12 @@
"""One Redis MGET per phase (admission, reservation, post-call) for the spend counters it reads, not one GET each."""
import asyncio
import itertools
from collections.abc import Iterator, Mapping, Sequence
from contextvars import ContextVar, Token
from dataclasses import dataclass
from types import MappingProxyType, TracebackType
from typing import Final
from typing import Final, TypeAlias
from pydantic import TypeAdapter
@ -22,6 +23,24 @@ from litellm.proxy.common_utils.user_api_key_cache import (
_CounterValues: Final = TypeAdapter(dict[str, float | None])
_NO_VALUES: Final[Mapping[str, float | None]] = MappingProxyType({})
SPEND_COUNTERS_TARGET: Final = "spend_counters"
_ticks: Final = itertools.count(1)
def spend_counter_tick() -> int:
"""The next number in a per-process sequence that orders spend counter reads and cold seeds."""
return next(_ticks)
@dataclass(frozen=True, slots=True)
class SpendCounterRead:
"""When a read of a spend counter went out and when its reply came back, as spend counter ticks."""
sent_at: int
returned_at: int
_NO_READS: Final[Mapping[str, SpendCounterRead]] = MappingProxyType({})
_PendingMGet: TypeAlias = tuple[BatchResult[Mapping[str, object]], int, "asyncio.Future[int]"]
@dataclass(frozen=True, slots=True)
@ -39,7 +58,17 @@ class SpendCounterBatch:
hook declares whatever is bound but unread, so whoever flushes first (the auth object prefetch, usually)
carries the spend counters in the same round trip."""
__slots__ = ("_fetched", "_inflight", "_keys", "_loaded", "_lock", "_open", "_redis_cache", "_request_batch")
__slots__ = (
"_fetched",
"_inflight",
"_keys",
"_loaded",
"_lock",
"_open",
"_reads",
"_redis_cache",
"_request_batch",
)
def __init__(self, redis_cache: RedisCache) -> None:
self._redis_cache: Final = redis_cache
@ -48,7 +77,8 @@ class SpendCounterBatch:
self._keys: frozenset[str] = frozenset()
self._fetched: frozenset[str] = frozenset()
self._loaded: Mapping[str, float | None] = _NO_VALUES
self._inflight: Final[list[BatchResult[Mapping[str, object]]]] = [] # mutable-ok: drained by _load
self._reads: Mapping[str, SpendCounterRead] = _NO_READS
self._inflight: Final[list[_PendingMGet]] = [] # mutable-ok: drained by _load
self._request_batch: Final[RedisBatch | None] = active_request_redis_batch(redis_cache)
if self._request_batch is not None:
self._request_batch.add_flush_hook(self._declare_pending)
@ -78,6 +108,10 @@ class SpendCounterBatch:
return None
return loaded[counter_key], True
def counter_read(self, counter_key: str) -> SpendCounterRead | None:
"""When the MGET behind this batch's value for the counter went out and came back, None without that value."""
return self._reads.get(counter_key) if self._open and counter_key in self._loaded else None
def record(self, counter_key: str, value: float) -> None:
"""A write returned the counter's new value; later reads in this scope see it instead of the MGET value."""
if not self._open:
@ -93,6 +127,7 @@ class SpendCounterBatch:
self._keys = self._keys - key
self._fetched = self._fetched - key
self._loaded = MappingProxyType({k: v for k, v in self._loaded.items() if k != counter_key})
self._reads = MappingProxyType({k: v for k, v in self._reads.items() if k != counter_key})
async def _load(self) -> Mapping[str, float | None]:
async with self._lock:
@ -103,10 +138,15 @@ class SpendCounterBatch:
pending: Final = self._keys - self._fetched
if pending:
self._fetched = self._fetched | pending
sent_at: Final = spend_counter_tick()
fetched: Final = await self._fetch(pending)
self._loaded = MappingProxyType({**fetched, **self._loaded})
self._merge(fetched, SpendCounterRead(sent_at=sent_at, returned_at=spend_counter_tick()))
return self._loaded
def _merge(self, fetched: Mapping[str, float | None], read: SpendCounterRead) -> None:
self._loaded = MappingProxyType({**fetched, **self._loaded})
self._reads = MappingProxyType({**dict.fromkeys(fetched, read), **self._reads})
def _declare_pending(self) -> None:
"""Flush hook: put every bound-but-unread counter on the request pipeline that is about to go out."""
if self._request_batch is None or not self._open:
@ -114,19 +154,28 @@ class SpendCounterBatch:
pending: Final = self._keys - self._fetched
if pending:
self._fetched = self._fetched | pending
sent_at: Final = spend_counter_tick()
with service_target(SPEND_COUNTERS_TARGET):
self._inflight.append(self._request_batch.mget(sorted(pending)))
read: Final = self._request_batch.mget(sorted(pending))
returned_at: Final[asyncio.Future[int]] = asyncio.get_running_loop().create_future()
read.on_settled(lambda _: returned_at.set_result(spend_counter_tick()))
self._inflight.append((read, sent_at, returned_at))
async def _collect_inflight(self) -> None:
results: Final = tuple(self._inflight)
reads: Final = tuple(self._inflight)
self._inflight.clear()
for result in results:
for read, sent_at, returned_at in reads:
try:
fetched: Mapping[str, float | None] = _CounterValues.validate_python(await result)
fetched: Mapping[str, float | None] = _CounterValues.validate_python(await read)
except Exception as e: # noqa: BLE001 # per-key reads take over and apply their own Redis fallback
verbose_proxy_logger.debug("spend counter batch read failed, falling back to per-key reads: %s", e)
continue
self._loaded = MappingProxyType({**fetched, **self._loaded})
self._merge(
fetched,
SpendCounterRead(
sent_at=sent_at, returned_at=returned_at.result() if returned_at.done() else spend_counter_tick()
),
)
async def _fetch(self, keys: frozenset[str]) -> Mapping[str, float | None]:
try:
@ -145,6 +194,14 @@ def active_spend_counter_batch() -> SpendCounterBatch | None:
return _active_batch.get()
def spend_counter_read(counter_key: str, sent_at: int) -> SpendCounterRead:
"""When this context's last read of the counter went out and came back: the open batch's MGET for it, else a read
that went out at ``sent_at`` and came back now."""
batch: Final = _active_batch.get()
batched: Final = batch.counter_read(counter_key) if batch is not None else None
return batched if batched is not None else SpendCounterRead(sent_at=sent_at, returned_at=spend_counter_tick())
class spend_counter_batch_scope:
"""Reads inside the scope share one MGET for the keys bound here or by ``bind_*`` calls inside it.
Opened inside a scope whose batch is still open, it binds into that batch so both phases share the MGET."""

View file

@ -1,17 +1,20 @@
import json
import os
import re
import uuid
from collections.abc import Callable, Mapping
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass
from typing import Final
import httpx
import pytest
from integration._support.client import Gateway, Scenario, eventually, object_value, string_value
from integration._support.database import read_rows
from integration._support.database import read_rows, write_rows
from integration._support.upstream import delete_scenario, register_scenario
from integration.cost_calculation.cost_tracking_case import JsonResponse
from pydantic import BaseModel, JsonValue
from redis import Redis
PROVIDER_AUTHORIZATION: Final = "Bearer integration-provider-key"
CANNED_CONTENT: Final = "Hello! This is a mock response from the fake OpenAI endpoint."
@ -639,3 +642,141 @@ def test_streamed_call_is_refused_as_json_once_over_budget(gateway: Gateway) ->
assert "data:" not in denied.text, denied.text
_assert_budget_refusal(denied, end_user, 0.05)
assert _observed_requests(upstream) == []
_COLD_BURST: Final = 40
_COLD_SEED_SPEND: Final = 6.0
_COLD_BUDGET: Final = 10.0
_CHAT_COST: Final = 20 * 0.001 + 20 * 0.002
def _serves_model(gateway: Gateway, model: str) -> bool:
response: Final = gateway.request("GET", "/model/info")
return response.status_code == 200 and any(
object_value(entry)["model_name"] == model for entry in object_value(response.json())["data"]
)
def _counter_cache() -> Redis:
return Redis(host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"]))
def _counter(cache: Redis, end_user: str) -> float | None:
raw: Final = cache.get(f"spend:end_user:{end_user}")
return None if raw is None else float(raw)
def _lose_counter(cache: Redis, end_user: str) -> None:
cache.delete(f"spend:end_user:{end_user}")
def _set_end_user_row_spend(end_user: str, spend: float) -> None:
write_rows('UPDATE "LiteLLM_EndUserTable" SET spend = %s WHERE user_id = %s', (str(spend), end_user))
def _await_row_spend(end_user: str, spend: float) -> None:
eventually(
lambda: _end_user_row(end_user),
lambda rows: len(rows) == 1 and float(str(rows[0]["spend"])) == pytest.approx(spend),
seconds=70,
)
def _refused_spend(denied: httpx.Response, end_user: str) -> float:
error: Final = _denial_error(denied)
assert error["type"] == "budget_exceeded", denied.text
matched: Final = _REFUSAL_MESSAGE.match(string_value(error["message"]))
assert matched is not None, denied.text
assert matched.group("user") == end_user
return float(matched.group("spend"))
@pytest.mark.parametrize("spelling", ("user", "responses_user", "messages_metadata_user_id"))
def test_overlapping_cold_starts_count_the_end_user_spend_once(gateway: Gateway, spelling: str) -> None:
chosen: Final = SPELLINGS[spelling]
with (
gateway.scenario() as scenario,
httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream,
_counter_cache() as cache,
):
deployment: Final = _deployment(scenario, chosen)
key: Final = scenario.key(models=[deployment.model])
end_user: Final = f"integration-end-user-{uuid.uuid4().hex}"
_new_customer(gateway, end_user, max_budget=_COLD_BUDGET, spend=_COLD_SEED_SPEND)
scenario.cleanups.callback(gateway.request, "POST", "/customer/delete", {"user_ids": [end_user]})
assert _customer_info(gateway, end_user).spend == _COLD_SEED_SPEND
assert _counter(cache, end_user) is None
upstream.delete("/__observations").raise_for_status()
with ThreadPoolExecutor(max_workers=_COLD_BURST) as pool:
burst: Final = tuple(
pool.map(
lambda index: _send(gateway, chosen, deployment, key, end_user, f"cold {index}"),
range(_COLD_BURST),
)
)
refused: Final = [response.text for response in burst if response.status_code != 200]
assert refused == [], refused
follow_up: Final = _send(gateway, chosen, deployment, key, end_user, "cold follow-up")
_assert_served(chosen, follow_up)
assert len(_observed_requests(upstream)) == _COLD_BURST + 1
expected_spend: Final = _COLD_SEED_SPEND + (_COLD_BURST + 1) * _CHAT_COST
_await_row_spend(end_user, expected_spend)
assert _customer_info(gateway, end_user).spend == pytest.approx(expected_spend)
counted: Final = eventually(
lambda: _counter(cache, end_user),
lambda value: value is not None and value == pytest.approx(expected_spend),
seconds=30,
return_last_on_timeout=True,
)
assert counted == pytest.approx(expected_spend)
def test_a_lost_counter_keeps_spend_this_proxy_has_not_written_yet(gateway: Gateway) -> None:
with gateway.scenario() as scenario, _counter_cache() as cache:
model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.25)
key: Final = scenario.key(models=[model])
end_user: Final = f"integration-end-user-{uuid.uuid4().hex}"
call_cost: Final = 20 * 0.001 + 20 * 0.25
_new_customer(gateway, end_user, max_budget=_COLD_BUDGET, spend=_COLD_SEED_SPEND)
scenario.cleanups.callback(gateway.request, "POST", "/customer/delete", {"user_ids": [end_user]})
served: Final = _chat(gateway, model, key, end_user)
assert served.status_code == 200, served.text
_await_row_spend(end_user, _COLD_SEED_SPEND + call_cost)
_set_end_user_row_spend(end_user, _COLD_SEED_SPEND)
_lose_counter(cache, end_user)
denied: Final = _chat(gateway, model, key, end_user)
assert _refused_spend(denied, end_user) == pytest.approx(_COLD_SEED_SPEND + call_cost)
def test_a_lost_counter_keeps_spend_another_proxy_has_not_written_yet(gateway: Gateway, peer: Gateway) -> None:
with gateway.scenario() as scenario, _counter_cache() as cache:
model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.149)
eventually(lambda: _serves_model(peer, model), bool, seconds=30)
key: Final = scenario.key(models=[model])
end_user: Final = f"integration-end-user-{uuid.uuid4().hex}"
call_cost: Final = 20 * 0.001 + 20 * 0.149
_new_customer(gateway, end_user, max_budget=_COLD_BUDGET, spend=_COLD_SEED_SPEND)
scenario.cleanups.callback(gateway.request, "POST", "/customer/delete", {"user_ids": [end_user]})
first: Final = _chat(peer, model, key, end_user)
assert first.status_code == 200, first.text
_await_row_spend(end_user, _COLD_SEED_SPEND + call_cost)
_set_end_user_row_spend(end_user, _COLD_SEED_SPEND)
_lose_counter(cache, end_user)
second: Final = _chat(peer, model, key, end_user)
assert second.status_code == 200, second.text
expected_spend: Final = _COLD_SEED_SPEND + 2 * call_cost
counted: Final = eventually(
lambda: _counter(cache, end_user),
lambda value: value is not None and value == pytest.approx(expected_spend),
seconds=30,
return_last_on_timeout=True,
)
denied: Final = _chat(gateway, model, key, end_user)
assert counted == pytest.approx(expected_spend)
assert _denial_error(denied)["type"] == "budget_exceeded", denied.text

View file

@ -22,6 +22,7 @@ Pins covered:
from __future__ import annotations
import asyncio
import threading
from collections.abc import Awaitable, Callable
from datetime import datetime
from typing import Final, Literal, cast
@ -36,10 +37,12 @@ from opentelemetry.trace import StatusCode
import litellm
import litellm.proxy.proxy_server as ps
from litellm._service_logger import ServiceLogging
from litellm._internal_context import current_service_target
from litellm._service_logger import ServiceLogging
from litellm.caching.dual_cache import DualCache
from litellm.caching.in_memory_cache import InMemoryCache
from litellm.caching.redis_cache import RedisCache, RedisCircuitBreaker
from litellm.caching.redis_batch import request_redis_batch_scope
from litellm.caching.redis_cache import RedisCache, RedisCircuitBreaker, RedisCircuitBreakerOpenError
from litellm.constants import GLOBAL_PROXY_SPEND_CACHE_KEY
from litellm.integrations.opentelemetry import OpenTelemetry, OpenTelemetryConfig
from litellm.models.end_user import LiteLLM_EndUserTable
@ -50,6 +53,7 @@ from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import evict_loca
from litellm.proxy.common_utils.cache_pydantic_utils import CacheCodec
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache, end_user_cache_key, tag_cache_key
from litellm.proxy.management_endpoints.team_endpoints import refresh_cached_team
from litellm.proxy.spend_tracking.spend_counter_batch import active_spend_counter_batch, spend_counter_batch_scope
from .conftest import normalize
@ -1805,10 +1809,10 @@ class _FakeRedisRunningIncrementIfPresent(FakeAsyncRedis):
class _RedisCacheOnClient(RedisCache):
def __init__(self, client: FakeAsyncRedis) -> None:
def __init__(self, client: FakeAsyncRedis, breaker: RedisCircuitBreaker | None = None) -> None:
self._injected_client: Final = client
self.namespace = None
self._circuit_breaker = RedisCircuitBreaker(failure_threshold=5, recovery_timeout=30)
self._circuit_breaker = breaker or RedisCircuitBreaker(failure_threshold=5, recovery_timeout=30)
self.service_logger_obj = ServiceLogging()
self.default_ttl = None
@ -1817,7 +1821,9 @@ class _RedisCacheOnClient(RedisCache):
class _FakeRedisHoldingOneReply(FakeAsyncRedis):
def __init__(self, server: FakeServer, held_increment: float, applied: asyncio.Event, released: asyncio.Event) -> None:
def __init__(
self, server: FakeServer, held_increment: float, applied: asyncio.Event, released: asyncio.Event
) -> None:
super().__init__(server=server)
self._held_increment: Final = held_increment
self._applied: Final = applied
@ -1970,9 +1976,7 @@ async def test_update_cache_traces_the_global_spend_redis_call_under_the_request
for span in exporter.get_finished_spans()
if span.name == "redis" and span.attributes is not None and span.parent is not None
)
assert redis_spans == (
("async_increment_if_present", expected_status, request_span.get_span_context().span_id),
)
assert redis_spans == (("async_increment_if_present", expected_status, request_span.get_span_context().span_id),)
class _UnwritableKeyUserApiKeyCache(UserApiKeyCache):
@ -2013,3 +2017,556 @@ async def test_update_cache_still_adds_spend_to_the_others_when_one_cached_objec
1.25,
10.25,
)
_COLD_SEED_SOURCE: Final = "end_user_id:customer-1"
class _SharedRedis:
"""The counter commands a cold start sends, yielding like a network call so overlapping requests interleave."""
circuit_breaker_closed: Final = True
def __init__(self) -> None:
self.values: Final[dict[str, float]] = {} # mutable-ok: stands in for the Redis keyspace
self.failing = False
async def async_batch_get_cache(self, key_list: list[str], **_: object) -> dict[str, float | None]:
await asyncio.sleep(0)
return {key: self.values.get(key) for key in key_list}
async def async_get_cache(self, key: str, **_: object) -> float | None:
await asyncio.sleep(0)
return self.values.get(key)
async def async_increment(self, key: str, value: float, **_: object) -> float:
await asyncio.sleep(0)
if self.failing:
raise ConnectionError("redis down")
self.values[key] = self.values.get(key, 0.0) + value
return self.values[key]
async def async_delete_cache(self, key: str) -> None:
await asyncio.sleep(0)
if not self.failing:
self.values.pop(key, None)
def _process(
monkeypatch: pytest.MonkeyPatch, redis: _SharedRedis | RedisCache | None, cached_spend: float
) -> DualCache:
counters: Final = DualCache(in_memory_cache=InMemoryCache(), redis_cache=cast(RedisCache, redis))
objects: Final = DualCache(in_memory_cache=InMemoryCache())
objects.in_memory_cache.set_cache(key=_COLD_SEED_SOURCE, value={"spend": cached_spend})
monkeypatch.setattr(ps, "spend_counter_cache", counters)
monkeypatch.setattr(ps, "user_api_key_cache", objects)
monkeypatch.setattr(ps, "prisma_client", object())
return counters
async def _cold_start(redis: _SharedRedis | RedisCache | None, counter: str) -> None:
with spend_counter_batch_scope(cast(RedisCache, redis) if redis is not None else None, frozenset((counter,))):
await ps.ensure_spend_counter_initialized(counter_key=counter, source_cache_key=_COLD_SEED_SOURCE)
async def _read_cold(counter: str) -> None:
batch: Final = active_spend_counter_batch()
assert batch is not None
assert await batch.read(counter) == (None, True)
@pytest.mark.asyncio
@pytest.mark.parametrize("with_redis", (True, False))
async def test_overlapping_cold_starts_in_one_process_add_its_cached_spend_once(
monkeypatch: pytest.MonkeyPatch, with_redis: bool
) -> None:
counter: Final = f"spend:end_user:once-{with_redis}"
redis: Final = _SharedRedis() if with_redis else None
counters: Final = _process(monkeypatch, redis, cached_spend=6.0)
await asyncio.gather(*(_cold_start(redis, counter) for _ in range(5)))
seeded: Final = redis.values.get(counter) if redis is not None else counters.in_memory_cache.get_cache(key=counter)
assert seeded == 6.0
@pytest.mark.asyncio
async def test_a_request_that_read_the_counter_before_this_process_seeded_it_does_not_add_again(
monkeypatch: pytest.MonkeyPatch,
) -> None:
counter: Final = "spend:end_user:stale-reader"
redis: Final = _SharedRedis()
_process(monkeypatch, redis, cached_spend=6.0)
read: Final = asyncio.Event()
async def _stale_request() -> None:
with spend_counter_batch_scope(cast(RedisCache, redis), frozenset((counter,))):
await _read_cold(counter)
read.set()
await seeded.wait()
await ps.ensure_spend_counter_initialized(counter_key=counter, source_cache_key=_COLD_SEED_SOURCE)
seeded: Final = asyncio.Event()
async def _first_request() -> None:
await read.wait()
await _cold_start(redis, counter)
seeded.set()
await asyncio.gather(_stale_request(), _first_request())
assert redis.values[counter] == 6.0
@pytest.mark.asyncio
async def test_a_counter_lost_again_after_this_process_seeded_it_is_seeded_again(
monkeypatch: pytest.MonkeyPatch,
) -> None:
counter: Final = "spend:end_user:lost-again"
redis: Final = _SharedRedis()
_process(monkeypatch, redis, cached_spend=6.0)
read: Final = asyncio.Event()
seeded: Final = asyncio.Event()
async def _stale_request() -> None:
with spend_counter_batch_scope(cast(RedisCache, redis), frozenset((counter,))):
await _read_cold(counter)
read.set()
await seeded.wait()
del redis.values[counter]
await ps.ensure_spend_counter_initialized(counter_key=counter, source_cache_key=_COLD_SEED_SOURCE)
async def _first_request() -> None:
await read.wait()
await _cold_start(redis, counter)
seeded.set()
await asyncio.gather(_stale_request(), _first_request())
assert redis.values[counter] == 6.0
@pytest.mark.asyncio
async def test_another_process_seeding_after_this_request_read_the_counter_still_sums(
monkeypatch: pytest.MonkeyPatch,
) -> None:
counter: Final = "spend:end_user:two-processes"
redis: Final = _SharedRedis()
read: Final = asyncio.Event()
a_seeded: Final = asyncio.Event()
async def _process_b() -> None:
with spend_counter_batch_scope(cast(RedisCache, redis), frozenset((counter,))):
await _read_cold(counter)
read.set()
await a_seeded.wait()
_process(monkeypatch, redis, cached_spend=5.0)
await ps.ensure_spend_counter_initialized(counter_key=counter, source_cache_key=_COLD_SEED_SOURCE)
async def _process_a() -> None:
await read.wait()
_process(monkeypatch, redis, cached_spend=3.0)
await _cold_start(redis, counter)
a_seeded.set()
await asyncio.gather(_process_b(), _process_a())
assert redis.values[counter] == 8.0
@pytest.mark.asyncio
async def test_seeds_that_keep_failing_fail_every_cold_start_and_the_next_cold_start_seeds(
monkeypatch: pytest.MonkeyPatch,
) -> None:
counter: Final = "spend:end_user:failed-seed"
redis: Final = _SharedRedis()
_process(monkeypatch, redis, cached_spend=6.0)
redis.failing = True
outcomes: Final = await asyncio.gather(*(_cold_start(redis, counter) for _ in range(3)), return_exceptions=True)
redis.failing = False
await _cold_start(redis, counter)
assert [type(outcome) for outcome in outcomes] == [ConnectionError] * 3
assert redis.values[counter] == 6.0
class _HeldReplyRedis(_SharedRedis):
"""Applies an increment at once but holds its reply until released, like a slow network round trip."""
def __init__(self) -> None:
super().__init__()
self.applied: Final = asyncio.Event()
self.release: Final = asyncio.Event()
async def async_increment(self, key: str, value: float, **_: object) -> float:
self.values[key] = self.values.get(key, 0.0) + value
total: Final = self.values[key]
self.applied.set()
await self.release.wait()
return total
@pytest.mark.asyncio
async def test_a_counter_deleted_before_an_earlier_add_returns_is_seeded_by_a_later_cold_reader(
monkeypatch: pytest.MonkeyPatch,
) -> None:
counter: Final = "spend:end_user:deleted-mid-add"
redis: Final = _HeldReplyRedis()
_process(monkeypatch, redis, cached_spend=6.0)
first: Final = asyncio.ensure_future(_cold_start(redis, counter))
await redis.applied.wait()
del redis.values[counter]
later: Final = asyncio.ensure_future(_cold_start(redis, counter))
for _ in range(20):
await asyncio.sleep(0)
redis.release.set()
await asyncio.gather(first, later)
assert redis.values[counter] == 6.0
def test_a_cold_start_on_another_event_loop_adds_on_its_own(monkeypatch: pytest.MonkeyPatch) -> None:
counter: Final = "spend:end_user:two-loops"
held: Final = threading.Event()
release: Final = threading.Event()
class _ThreadHeldRedis(_SharedRedis):
async def async_increment(self, key: str, value: float, **_: object) -> float:
if threading.current_thread() is not threading.main_thread():
held.set()
while not release.is_set():
await asyncio.sleep(0.001)
return await super().async_increment(key, value)
redis: Final = _ThreadHeldRedis()
_process(monkeypatch, redis, cached_spend=6.0)
other_loop: Final = threading.Thread(target=lambda: asyncio.run(_cold_start(redis, counter)), daemon=True)
other_loop.start()
assert held.wait(5)
try:
asyncio.run(_cold_start(redis, counter))
finally:
release.set()
other_loop.join(5)
assert redis.values[counter] == 12.0
@pytest.mark.asyncio
async def test_a_request_that_read_the_counter_cold_after_this_process_seeded_it_still_adds_its_cached_spend(
monkeypatch: pytest.MonkeyPatch,
) -> None:
counter: Final = "spend:end_user:read-after-seed"
redis: Final = _SharedRedis()
_process(monkeypatch, redis, cached_spend=6.0)
opened: Final = asyncio.Event()
seeded: Final = asyncio.Event()
async def _late_reader() -> None:
with spend_counter_batch_scope(cast(RedisCache, redis), frozenset((counter,))):
opened.set()
await seeded.wait()
del redis.values[counter]
await _read_cold(counter)
redis.values[counter] = 0.5
await ps.ensure_spend_counter_initialized(counter_key=counter, source_cache_key=_COLD_SEED_SOURCE)
async def _first_request() -> None:
await opened.wait()
await _cold_start(redis, counter)
seeded.set()
await asyncio.gather(_late_reader(), _first_request())
assert redis.values[counter] == 6.5
@pytest.mark.asyncio
async def test_a_request_whose_pipelined_read_returned_before_this_process_seeded_the_counter_does_not_add_again(
monkeypatch: pytest.MonkeyPatch,
) -> None:
counter: Final = "spend:end_user:pipelined-reader"
client: Final = FakeAsyncRedis(server=FakeServer())
redis: Final = _RedisCacheOnClient(client)
_process(monkeypatch, redis, cached_spend=6.0)
read: Final = asyncio.Event()
seeded: Final = asyncio.Event()
async def _pipelined_request() -> None:
with request_redis_batch_scope() as batches, spend_counter_batch_scope(redis, frozenset((counter,))):
await batches.batch(redis).flush()
read.set()
await seeded.wait()
await ps.ensure_spend_counter_initialized(counter_key=counter, source_cache_key=_COLD_SEED_SOURCE)
async def _first_request() -> None:
await read.wait()
await _cold_start(redis, counter)
seeded.set()
await asyncio.gather(_pipelined_request(), _first_request())
assert float(await client.get(counter) or 0) == 6.0
class _DroppedFirstIncrementRedis(_SharedRedis):
"""Resets the connection that carries the first increment and answers every later one."""
def __init__(self) -> None:
super().__init__()
self.dropped = False
async def async_increment(self, key: str, value: float, **_: object) -> float:
if not self.dropped:
self.dropped = True
await asyncio.sleep(0)
raise ConnectionError("connection reset by peer")
return await super().async_increment(key, value)
@pytest.mark.asyncio
async def test_requests_that_joined_a_seed_whose_connection_reset_seed_again_and_only_its_starter_fails(
monkeypatch: pytest.MonkeyPatch,
) -> None:
counter: Final = "spend:end_user:reset-seed"
redis: Final = _DroppedFirstIncrementRedis()
_process(monkeypatch, redis, cached_spend=6.0)
outcomes: Final = await asyncio.gather(*(_cold_start(redis, counter) for _ in range(4)), return_exceptions=True)
failures: Final = [outcome for outcome in outcomes if outcome is not None]
assert [type(failure) for failure in failures] == [ConnectionError], outcomes
assert redis.values[counter] == 6.0
class _StalledFirstIncrementRedis(_SharedRedis):
"""Never answers the first increment until released, like a connection that stopped responding."""
def __init__(self) -> None:
super().__init__()
self.stalled: Final = asyncio.Event()
self.release: Final = asyncio.Event()
async def async_increment(self, key: str, value: float, **_: object) -> float:
if not self.stalled.is_set():
self.stalled.set()
await self.release.wait()
raise TimeoutError("redis did not answer")
return await super().async_increment(key, value)
@pytest.mark.asyncio
async def test_cold_starts_stop_waiting_on_a_stalled_seed_and_add_their_own(monkeypatch: pytest.MonkeyPatch) -> None:
counter: Final = "spend:end_user:stalled-seed"
redis: Final = _StalledFirstIncrementRedis()
_process(monkeypatch, redis, cached_spend=6.0)
stalled: Final = asyncio.ensure_future(_cold_start(redis, counter))
await redis.stalled.wait()
try:
await asyncio.wait_for(asyncio.gather(*(_cold_start(redis, counter) for _ in range(3))), timeout=10)
seeded_while_stalled: Final = redis.values.get(counter)
finally:
redis.release.set()
with pytest.raises(TimeoutError):
await stalled
assert seeded_while_stalled == 18.0
class _FakeRedisHoldingIncrements(FakeAsyncRedis):
def __init__(self, server: FakeServer) -> None:
super().__init__(server=server)
self.holding: Final = asyncio.Event()
self.release: Final = asyncio.Event()
async def incrbyfloat(self, name: str, amount: float = 1.0) -> float:
self.holding.set()
await self.release.wait()
return await super().incrbyfloat(name, amount)
@pytest.mark.asyncio
async def test_a_cold_start_after_the_redis_breaker_opened_fails_fast_instead_of_using_an_add_in_flight(
monkeypatch: pytest.MonkeyPatch,
) -> None:
counter: Final = "spend:end_user:breaker-open"
breaker: Final = RedisCircuitBreaker(failure_threshold=1, recovery_timeout=30)
client: Final = _FakeRedisHoldingIncrements(FakeServer())
redis: Final = _RedisCacheOnClient(client, breaker)
_process(monkeypatch, redis, cached_spend=6.0)
first: Final = asyncio.ensure_future(_cold_start(redis, counter))
await client.holding.wait()
breaker.record_failure()
later: Final = asyncio.ensure_future(_cold_start(redis, counter))
for _ in range(100):
await asyncio.sleep(0)
client.release.set()
outcomes: Final = await asyncio.gather(first, later, return_exceptions=True)
assert outcomes[0] is None and isinstance(outcomes[1], RedisCircuitBreakerOpenError), outcomes
class _HeldFirstMGetRedis(_SharedRedis):
"""Reads the first MGET's keys at once but holds its reply until released, like a slow network round trip."""
def __init__(self) -> None:
super().__init__()
self.mget_read: Final = asyncio.Event()
self.release_mget: Final = asyncio.Event()
async def async_batch_get_cache(self, key_list: list[str], **_: object) -> dict[str, float | None]:
values: Final = {key: self.values.get(key) for key in key_list}
if not self.mget_read.is_set():
self.mget_read.set()
await self.release_mget.wait()
return values
@pytest.mark.asyncio
async def test_a_seed_that_lands_while_a_cold_read_is_on_its_way_back_is_not_added_again(
monkeypatch: pytest.MonkeyPatch,
) -> None:
counter: Final = "spend:end_user:seed-during-read"
redis: Final = _HeldFirstMGetRedis()
_process(monkeypatch, redis, cached_spend=6.0)
async def _first_request() -> None:
await redis.mget_read.wait()
await _cold_start(redis, counter)
redis.release_mget.set()
await asyncio.gather(_cold_start(redis, counter), _first_request())
assert redis.values[counter] == 6.0
class _HeldReadRedis(_SharedRedis):
"""Answers an MGET with the values its keys had when it arrived, holding the reply while ``hold_reads`` is set."""
def __init__(self) -> None:
super().__init__()
self.hold_reads = False
self.read_arrived: Final = asyncio.Event()
self.release_read: Final = asyncio.Event()
async def async_batch_get_cache(self, key_list: list[str], **_: object) -> dict[str, float | None]:
values: Final = {key: self.values.get(key) for key in key_list}
if self.hold_reads:
self.read_arrived.set()
await self.release_read.wait()
return values
@pytest.mark.asyncio
async def test_a_seed_that_lands_while_a_request_re_reads_the_counter_is_not_added_again(
monkeypatch: pytest.MonkeyPatch,
) -> None:
counter: Final = "spend:end_user:seed-during-reread"
redis: Final = _HeldReadRedis()
_process(monkeypatch, redis, cached_spend=6.0)
read: Final = asyncio.Event()
seeded: Final = asyncio.Event()
async def _stale_request() -> None:
with spend_counter_batch_scope(cast(RedisCache, redis), frozenset((counter,))):
await _read_cold(counter)
read.set()
await seeded.wait()
del redis.values[counter]
redis.hold_reads = True
await ps.ensure_spend_counter_initialized(counter_key=counter, source_cache_key=_COLD_SEED_SOURCE)
async def _first_request() -> None:
await read.wait()
await _cold_start(redis, counter)
seeded.set()
async def _request_seeding_during_the_re_read() -> None:
await redis.read_arrived.wait()
redis.hold_reads = False
await _cold_start(redis, counter)
redis.release_read.set()
await asyncio.gather(_stale_request(), _first_request(), _request_seeding_during_the_re_read())
assert redis.values[counter] == 6.0
class _HeldGetRedis(_SharedRedis):
"""Answers the first GET with the value its key had when it arrived, holding the reply until released."""
def __init__(self) -> None:
super().__init__()
self.get_arrived: Final = asyncio.Event()
self.release_get: Final = asyncio.Event()
async def async_get_cache(self, key: str, **_: object) -> float | None:
value: Final = self.values.get(key)
if not self.get_arrived.is_set():
self.get_arrived.set()
await self.release_get.wait()
return value
@pytest.mark.asyncio
async def test_a_seed_that_lands_while_an_unbatched_cold_read_is_on_its_way_back_is_not_added_again(
monkeypatch: pytest.MonkeyPatch,
) -> None:
counter: Final = "spend:end_user:seed-during-unbatched-read"
redis: Final = _HeldGetRedis()
_process(monkeypatch, redis, cached_spend=6.0)
async def _first_request() -> None:
await redis.get_arrived.wait()
await _cold_start(redis, counter)
redis.release_get.set()
await asyncio.gather(
ps.ensure_spend_counter_initialized(counter_key=counter, source_cache_key=_COLD_SEED_SOURCE),
_first_request(),
)
assert redis.values[counter] == 6.0
class _QueuedReadRedis(_HeldReplyRedis):
"""Holds the first MGET before Redis runs it, while increments apply at once and hold their replies."""
def __init__(self) -> None:
super().__init__()
self.read_sent: Final = asyncio.Event()
self.run_read: Final = asyncio.Event()
async def async_batch_get_cache(self, key_list: list[str], **_: object) -> dict[str, float | None]:
if not self.read_sent.is_set():
self.read_sent.set()
await self.run_read.wait()
return {key: self.values.get(key) for key in key_list}
@pytest.mark.asyncio
async def test_a_cold_read_sent_before_an_add_started_does_not_join_it_when_the_counter_was_deleted_after_it_landed(
monkeypatch: pytest.MonkeyPatch,
) -> None:
counter: Final = "spend:end_user:deleted-while-read-queued"
redis: Final = _QueuedReadRedis()
_process(monkeypatch, redis, cached_spend=6.0)
reader: Final = asyncio.ensure_future(_cold_start(redis, counter))
await redis.read_sent.wait()
first: Final = asyncio.ensure_future(_cold_start(redis, counter))
await redis.applied.wait()
del redis.values[counter]
redis.run_read.set()
for _ in range(20):
await asyncio.sleep(0)
redis.release.set()
await asyncio.gather(reader, first)
assert redis.values.get(counter) == 6.0