From 0ad56125d279ba0d63ea41714acbb3e22c8a099a Mon Sep 17 00:00:00 2001 From: yucheng Date: Mon, 2 Nov 2026 15:51:37 -0800 Subject: [PATCH] fix(proxy): set a missing cold counter with SET NX instead of a timed re-read A request whose cold read may predate this process's add now sends SET NX, EXPIRE and GET in one pipeline, the same shape as main's INCRBYFLOAT and EXPIRE, so it sends one write and no read, cancels nothing, and needs no second time limit It adds on top only when the counter holds less than what the landed add should have left, which keeps main's sum when a counter was lost and re-created below that The wait on other requests' adds now shares one limit across a request's counters, and the cold check read goes back to main's decorated helper so its span and service target stay spend_counters --- litellm/caching/redis_cache.py | 24 +- litellm/proxy/db/spend_counter_reseed.py | 91 ++-- litellm/proxy/proxy_server.py | 57 ++- .../spend_tracking/spend_counter_batch.py | 3 +- tests/unit/caching/test_redis_cache.py | 50 +++ .../proxy/proxy_server/test_spend_counters.py | 418 +++++++++++++----- 6 files changed, 475 insertions(+), 168 deletions(-) diff --git a/litellm/caching/redis_cache.py b/litellm/caching/redis_cache.py index 3b90cce211d..1e469e26990 100644 --- a/litellm/caching/redis_cache.py +++ b/litellm/caching/redis_cache.py @@ -1473,15 +1473,29 @@ class RedisCache(BaseCache): await _redis_client.expire(key, ttl) return float(result) + async def _set_if_missing_with_ttl(self, key: str, value: float, ttl: int | None) -> float: + """SET NX, its TTL command and a GET in one round trip; the value the key holds afterwards.""" + async with self._async_commands().pipeline(transaction=False) as pipe: + pipe.set(name=key, value=value, nx=True) + if ttl is not None: + pipe.expire(key, ttl) + pipe.get(key) + replies: Final[Sequence[object]] = await pipe.execute() # pyright: ignore[reportUnknownVariableType, reportUnknownMemberType] # redis generics + current: Final = _LUA_OPTIONAL_TOTAL.validate_python(replies[-1]) + return value if current is None else current + @_redis_circuit_breaker_guard async def async_increment( self, - key, + key: str, value: float, ttl: int | None = None, parent_otel_span: Span | None = None, refresh_ttl: bool = False, + only_if_missing: bool = False, ) -> float: + """Add ``value`` to the key and return its new value. With ``only_if_missing``, a key that exists keeps its + value, which is returned, and only the TTL is refreshed.""" from redis.asyncio import Redis _redis_client: Final[Redis] = self.init_async_client() @@ -1489,8 +1503,12 @@ class RedisCache(BaseCache): _used_ttl: Final = self.get_ttl(ttl=ttl) key = self.check_and_fix_namespace(key=key) try: - result: Final = await self._incrbyfloat_with_ttl( - _redis_client, key=key, value=value, ttl=_used_ttl, refresh_ttl=refresh_ttl + result: Final = await ( + self._set_if_missing_with_ttl(key=key, value=value, ttl=_used_ttl) + if only_if_missing + else self._incrbyfloat_with_ttl( + _redis_client, key=key, value=value, ttl=_used_ttl, refresh_ttl=refresh_ttl + ) ) ## LOGGING ## diff --git a/litellm/proxy/db/spend_counter_reseed.py b/litellm/proxy/db/spend_counter_reseed.py index f47dc17bbd6..8a0d413bdbc 100644 --- a/litellm/proxy/db/spend_counter_reseed.py +++ b/litellm/proxy/db/spend_counter_reseed.py @@ -16,6 +16,7 @@ import asyncio import functools from collections import OrderedDict from collections.abc import Awaitable, Callable, Mapping +from contextvars import ContextVar from dataclasses import dataclass from datetime import datetime, timezone from types import MappingProxyType @@ -30,7 +31,6 @@ from litellm.proxy.db.db_lookup_gate import bounded_db_lookup, db_lookup_gate from litellm.proxy.spend_tracking.spend_counter_batch import ( SPEND_COUNTERS_TARGET, SpendCounterRead, - forget_spend_counter, read_batched_spend_counter, record_spend_counter_value, spend_counter_tick, @@ -75,11 +75,12 @@ _WINDOW_SPEND_LOG_FIELDS: Final[Mapping[str, str]] = MappingProxyType( @dataclass(slots=True) class _CachedSpendSeed: - """This process adding its cached spend to one cold counter: the add in flight, the tick it started at, and the - tick the last one landed at.""" + """This process adding its cached spend to one cold counter: the add in flight, the tick it started at, the tick + the last one landed at, and the least the counter held when that one landed.""" started_at: int = 0 landed_at: int = 0 + landed_spend: float = 0.0 in_flight: asyncio.Future[float | None] | None = None def on_this_loop(self) -> bool: @@ -107,6 +108,18 @@ def _cached_spend_seed(spend_counter_cache: "DualCache", counter_key: str) -> _C return _CachedSpendSeed() +_cold_seed_waited: Final[ContextVar[float]] = ContextVar("cold_spend_seed_waited", default=0.0) + + +async def _wait_for_cold_seed(in_flight: asyncio.Future[float | None]) -> None: + """Wait on ``in_flight`` for what this context has left of ``SPEND_COUNTER_COLD_SEED_JOIN_SECONDS``.""" + loop: Final = asyncio.get_running_loop() + waited: Final = _cold_seed_waited.get() + started: Final = loop.time() + await asyncio.wait((in_flight,), timeout=max(0.0, SPEND_COUNTER_COLD_SEED_JOIN_SECONDS - waited)) + _cold_seed_waited.set(waited + loop.time() - started) + + def _redis_breaker_closed(spend_counter_cache: "DualCache") -> bool: redis_cache: Final = spend_counter_cache.redis_cache return redis_cache is None or redis_cache.circuit_breaker_closed @@ -161,56 +174,66 @@ class SpendCounterReseed: spend_counter_cache: "DualCache", counter_key: str, read: SpendCounterRead, + spend: float, add: Callable[[], Awaitable[float | None]], - is_warm: Callable[[], Awaitable[bool]], + set_if_missing: Callable[[], Awaitable[float | None]], ) -> None: - """Run ``add``, this process adding its cached spend to a cold counter, once for all of this process's - requests that read the counter cold before that add landed. A request joins an add that started after its read - came back and gets its result, and starts or joins another add when that one fails or is cancelled. When an add - landed after its read went out, or one that started before its read came back finishes, the request re-reads - the counter and adds again only if the counter is gone. A request waits on other requests' adds for at most - ``SPEND_COUNTER_COLD_SEED_JOIN_SECONDS`` in all, has that long again to finish a re-read, and then adds its own. - A request on another event loop than the add in flight, one that finds the Redis circuit breaker not closed, and - each other process, still add their own cached spend.""" + """Run ``add``, this process adding ``spend`` to a cold counter, once for all of this process's requests that + read the counter cold before that add landed. A request joins an add that started after its read came back and + gets its result, and starts or joins another add when that one fails or is cancelled. A request whose read went + out before an add landed, and one still waiting on an add after ``SPEND_COUNTER_COLD_SEED_JOIN_SECONDS`` across + its counters, runs ``set_if_missing`` in place of ``add``, then runs ``add`` too if the counter holds less than + the spend it should already hold. A request on another event loop than the add in flight, one that finds the + Redis circuit breaker not closed, and each other process, still run ``add``.""" seed: Final = _cached_spend_seed(spend_counter_cache, counter_key) - loop: Final = asyncio.get_running_loop() - join_by: Final = loop.time() + SPEND_COUNTER_COLD_SEED_JOIN_SECONDS - re_read_by: Final = join_by + SPEND_COUNTER_COLD_SEED_JOIN_SECONDS - async def add_unless_covered(cold: SpendCounterRead) -> float | None: + async def add_spend() -> float | None: + added: Final = await add() + seed.landed_spend = spend + return added + + async def add_if_short() -> float | None: + least: Final = min(spend, seed.landed_spend) if seed.landed_at >= read.sent_at else spend + current: Final = await set_if_missing() + if current is not None and current < least: + return await add_spend() + seed.landed_spend = least + return current + + async def add_unless_covered() -> float | None: in_flight: Final = seed.in_flight if not seed.on_this_loop() or not _redis_breaker_closed(spend_counter_cache): return await add() if in_flight is None: - if seed.landed_at < cold.sent_at: - return await seed.run(add) - forget_spend_counter(counter_key) - sent_at: Final = spend_counter_tick() - try: - warm: Final = await asyncio.wait_for(is_warm(), timeout=max(0.0, re_read_by - loop.time())) - except asyncio.TimeoutError: - return await seed.run(add) - if warm: - return None - return await add_unless_covered(SpendCounterRead(sent_at=sent_at, returned_at=spend_counter_tick())) - covers: Final = seed.started_at > cold.returned_at - await asyncio.wait((in_flight,), timeout=max(0.0, join_by - loop.time())) + return await seed.run(add_spend if seed.landed_at < read.sent_at else add_if_short) + covers: Final = seed.started_at > read.returned_at + await _wait_for_cold_seed(in_flight) if not in_flight.done(): - return await seed.run(add) + return await seed.run(add_if_short) if covers and not in_flight.cancelled(): return in_flight.result() - return await add_unless_covered(cold) + return await add_unless_covered() - added: Final = await add_unless_covered(read) + added: Final = await add_unless_covered() if added is not None: record_spend_counter_value(counter_key, added) @staticmethod @with_service_target(SPEND_COUNTERS_TARGET) - async def increment_in_memory(spend_counter_cache: "DualCache", counter_key: str, increment: float) -> float | None: - """Apply local deltas after an in-flight reseed establishes the spend balance.""" + async def increment_in_memory( + spend_counter_cache: "DualCache", counter_key: str, increment: float, only_if_missing: bool = False + ) -> float | None: + """Apply local deltas after an in-flight reseed establishes the spend balance. With ``only_if_missing``, a + counter that exists keeps its value, which is returned.""" lock: Final = await SpendCounterReseed._get_lock(counter_key) async with lock: + current: Final[float | None] = ( # pyright: ignore[reportUnknownVariableType] # untyped in-memory cache + spend_counter_cache.in_memory_cache.get_cache(key=counter_key) # pyright: ignore[reportUnknownMemberType] # untyped in-memory cache + if only_if_missing + else None + ) + if current is not None: + return float(current) return await spend_counter_cache.async_increment_cache( key=counter_key, value=increment, local_only=True, refresh_ttl=True ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 741f7d58f1d..7c1676e22c1 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -930,7 +930,6 @@ from litellm.proxy.spend_tracking.spend_capture_rate import ( from litellm.proxy.spend_tracking.spend_counter_batch import ( SPEND_COUNTERS_TARGET, PendingSpendIncrement, - SpendCounterRead, active_spend_counter_batch, forget_spend_counter, post_call_counter_keys, @@ -4087,8 +4086,9 @@ async def ensure_spend_counter_initialized( counter_key: str, source_cache_key: str | list[str], ) -> None: - read: Final = await _cold_spend_counter_read(counter_key) - if read is not None: + is_warm: Final = await _is_spend_counter_cache_warm(counter_key=counter_key) + if is_warm is False: + read: Final = spend_counter_read(counter_key) # Shares the per-counter lock with get_current_spend. db_spend: Final = await SpendCounterReseed.coalesced( prisma_client=prisma_client, @@ -4104,8 +4104,11 @@ async def ensure_spend_counter_initialized( spend_counter_cache=spend_counter_cache, counter_key=counter_key, read=read, + spend=base_spend, add=lambda: increment_spend_counter_cache(counter_key=counter_key, increment=base_spend), - is_warm=lambda: _is_spend_counter_warm_outside_batch(counter_key), + set_if_missing=lambda: increment_spend_counter_cache( + counter_key=counter_key, increment=base_spend, only_if_missing=True + ), ) @@ -4160,22 +4163,14 @@ _ensure_window_spend_counter_initialized: Final = ensure_window_spend_counter_in @with_service_target(SPEND_COUNTERS_TARGET) async def _is_spend_counter_cache_warm(counter_key: str) -> bool: - return await _cold_spend_counter_read(counter_key) is None - - -async def _cold_spend_counter_read(counter_key: str) -> SpendCounterRead | None: - """When the read that found the counter missing went out and came back, None when the counter has a value. A read - outside the batch counts as now, since ``SpendCounterReseed.coalesced`` reads the counter again after it.""" batched: Final = await read_batched_spend_counter(counter_key) - if batched is not None and batched[0] is not None: - spend_counter_cache.in_memory_cache.set_cache(key=counter_key, value=batched[0]) - return None - if batched is None and await _is_spend_counter_warm_outside_batch(counter_key): - return None - return spend_counter_read(counter_key) + if batched is not None: + batched_value, _ = batched + if batched_value is None: + return False + spend_counter_cache.in_memory_cache.set_cache(key=counter_key, value=batched_value) + return True - -async def _is_spend_counter_warm_outside_batch(counter_key: str) -> bool: if spend_counter_cache.redis_cache is not None: try: current_value: Final[object] = await spend_counter_cache.redis_cache.async_get_cache( @@ -4217,13 +4212,24 @@ async def refresh_spend_counter_ttl(counter_key: str) -> bool: @with_service_target(SPEND_COUNTERS_TARGET) -async def increment_spend_counter_cache(counter_key: str, increment: float) -> float | None: +async def increment_spend_counter_cache( + counter_key: str, increment: float, only_if_missing: bool = False +) -> float | None: if spend_counter_cache.redis_cache is not None: try: - current_value: Final = await spend_counter_cache.redis_cache.async_increment( - key=counter_key, - value=increment, - refresh_ttl=True, + current_value: Final = await ( + spend_counter_cache.redis_cache.async_increment( + key=counter_key, + value=increment, + refresh_ttl=True, + only_if_missing=True, + ) + if only_if_missing + else spend_counter_cache.redis_cache.async_increment( + key=counter_key, + value=increment, + refresh_ttl=True, + ) ) except Exception: await invalidate_spend_counter(counter_key=counter_key) @@ -4236,7 +4242,10 @@ async def increment_spend_counter_cache(counter_key: str, increment: float) -> f return current_value return await SpendCounterReseed.increment_in_memory( - spend_counter_cache=spend_counter_cache, counter_key=counter_key, increment=increment + spend_counter_cache=spend_counter_cache, + counter_key=counter_key, + increment=increment, + only_if_missing=only_if_missing, ) diff --git a/litellm/proxy/spend_tracking/spend_counter_batch.py b/litellm/proxy/spend_tracking/spend_counter_batch.py index 55c4fab6893..8ff709086bb 100644 --- a/litellm/proxy/spend_tracking/spend_counter_batch.py +++ b/litellm/proxy/spend_tracking/spend_counter_batch.py @@ -188,7 +188,8 @@ def active_spend_counter_batch() -> SpendCounterBatch | None: def spend_counter_read(counter_key: str) -> SpendCounterRead: - """When this context's read of the counter went out and came back: the open batch's MGET for it, else now.""" + """When this context's read of the counter went out and came back: the open batch's MGET for it, else now, since + ``SpendCounterReseed.coalesced`` reads the counter again after a read outside the batch.""" batch: Final = _active_batch.get() batched: Final = batch.counter_read(counter_key) if batch is not None else None if batched is not None: diff --git a/tests/unit/caching/test_redis_cache.py b/tests/unit/caching/test_redis_cache.py index 2bdc7b9549b..356a1f34985 100644 --- a/tests/unit/caching/test_redis_cache.py +++ b/tests/unit/caching/test_redis_cache.py @@ -8,6 +8,7 @@ from typing import Final, cast from unittest.mock import AsyncMock, MagicMock, patch import pytest +from fakeredis import FakeAsyncRedis, FakeServer import litellm from litellm._service_logger import ServiceLogging @@ -1474,6 +1475,55 @@ async def test_async_increment_pipelines_the_ttl_command( assert client.round_trips == expected_round_trips +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("existing", "expected"), + [ + pytest.param(None, 1.5, id="missing key: set to the value"), + pytest.param(0.25, 0.25, id="key below the value: kept"), + pytest.param(4.0, 4.0, id="key above the value: kept"), + ], +) +async def test_async_increment_only_if_missing_sets_a_missing_key_and_refreshes_the_ttl( + monkeypatch, redis_no_ping, existing, expected +): + monkeypatch.setenv("REDIS_HOST", "https://my-test-host") + redis_cache = RedisCache(namespace="ns") + client = FakeAsyncRedis(server=FakeServer()) + if existing is not None: + await client.set("ns:spend:key:k", existing, ex=5) + + with patch.object(redis_cache, "init_async_client", return_value=client): + current = await redis_cache.async_increment( + key="spend:key:k", value=1.5, ttl=60, refresh_ttl=True, only_if_missing=True + ) + after_add = await redis_cache.async_increment(key="spend:key:k", value=2.0, ttl=60, refresh_ttl=True) + + assert current == expected + assert after_add == expected + 2.0 + assert await client.ttl("ns:spend:key:k") == 60 + + +@pytest.mark.asyncio +async def test_async_increment_only_if_missing_reports_a_redis_failure_like_an_increment(monkeypatch, redis_no_ping): + monkeypatch.setenv("REDIS_HOST", "https://my-test-host") + redis_cache = RedisCache(namespace="ns") + client = MagicMock() + client.pipeline.side_effect = ConnectionError("connection reset") + failure_hook = AsyncMock() + + with ( + patch.object(redis_cache, "init_async_client", return_value=client), + patch.object(redis_cache.service_logger_obj, "async_service_failure_hook", failure_hook), + pytest.raises(ConnectionError), + ): + await redis_cache.async_increment(key="spend:key:k", value=1.5, ttl=60, refresh_ttl=True, only_if_missing=True) + await asyncio.sleep(0) + + assert failure_hook.await_args.kwargs["call_type"] == "async_increment" + assert redis_cache._circuit_breaker._hard_failure_count == 1 + + class _SetRecordingPipeline: def __init__(self) -> None: self.sets: list[tuple[str, str, timedelta | None]] = [] diff --git a/tests/unit/proxy/proxy_server/test_spend_counters.py b/tests/unit/proxy/proxy_server/test_spend_counters.py index c9342a1b895..e4a37721bba 100644 --- a/tests/unit/proxy/proxy_server/test_spend_counters.py +++ b/tests/unit/proxy/proxy_server/test_spend_counters.py @@ -25,7 +25,7 @@ import asyncio import threading from collections.abc import Awaitable, Callable from datetime import datetime -from typing import ClassVar, Final, Literal, cast +from typing import Final, Literal, cast from unittest.mock import AsyncMock, MagicMock, call import pytest @@ -37,13 +37,13 @@ from opentelemetry.trace import StatusCode import litellm import litellm.proxy.proxy_server as ps -from litellm._internal_context import current_service_target +from litellm._internal_context import current_service_target, service_target from litellm._service_logger import ServiceLogging from litellm.caching.dual_cache import DualCache from litellm.caching.in_memory_cache import InMemoryCache from litellm.caching.redis_batch import request_redis_batch_scope from litellm.caching.redis_cache import RedisCache, RedisCircuitBreaker, RedisCircuitBreakerOpenError -from litellm.constants import GLOBAL_PROXY_SPEND_CACHE_KEY +from litellm.constants import AUTH_OBJECTS_TARGET, GLOBAL_PROXY_SPEND_CACHE_KEY from litellm.integrations.opentelemetry import OpenTelemetry, OpenTelemetryConfig from litellm.models.end_user import LiteLLM_EndUserTable from litellm.models.tag import LiteLLM_TagTable @@ -55,6 +55,7 @@ from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache, end_u from litellm.proxy.db import spend_counter_reseed from litellm.proxy.management_endpoints.team_endpoints import refresh_cached_team from litellm.proxy.spend_tracking.spend_counter_batch import ( + SPEND_COUNTERS_TARGET, active_spend_counter_batch, read_batched_spend_counter, spend_counter_batch_scope, @@ -2044,12 +2045,16 @@ class _SharedRedis: await asyncio.sleep(0) return self.values.get(key) - async def async_increment(self, key: str, value: float, **_: object) -> float: + def _apply(self, key: str, value: float, only_if_missing: bool = False, **_: object) -> float: + if not only_if_missing or key not in self.values: + self.values[key] = self.values.get(key, 0.0) + value + return self.values[key] + + async def async_increment(self, key: str, value: float, **kwargs: bool) -> float: await asyncio.sleep(0) if self.failing: raise ConnectionError("redis down") - self.values[key] = self.values.get(key, 0.0) + value - return self.values[key] + return self._apply(key, value, **kwargs) async def async_delete_cache(self, key: str) -> None: await asyncio.sleep(0) @@ -2091,7 +2096,9 @@ async def test_overlapping_cold_starts_in_one_process_add_its_cached_spend_once( await asyncio.gather(*(_cold_start(redis, counter) for _ in range(5))) - seeded: Final = redis.values.get(counter) if redis is not None else counters.in_memory_cache.get_cache(key=counter) + seeded: Final[float | None] = ( + redis.values.get(counter) if redis is not None else counters.in_memory_cache.get_cache(key=counter) + ) assert seeded == 6.0 @@ -2205,9 +2212,8 @@ class _HeldReplyRedis(_SharedRedis): self.applied: Final = asyncio.Event() self.release: Final = asyncio.Event() - async def async_increment(self, key: str, value: float, **_: object) -> float: - self.values[key] = self.values.get(key, 0.0) + value - total: Final = self.values[key] + async def async_increment(self, key: str, value: float, **kwargs: bool) -> float: + total: Final = self._apply(key, value, **kwargs) self.applied.set() await self.release.wait() return total @@ -2240,12 +2246,12 @@ def test_a_cold_start_on_another_event_loop_adds_on_its_own(monkeypatch: pytest. release: Final = threading.Event() class _ThreadHeldRedis(_SharedRedis): - async def async_increment(self, key: str, value: float, **_: object) -> float: + async def async_increment(self, key: str, value: float, **kwargs: bool) -> float: if threading.current_thread() is not threading.main_thread(): held.set() while not release.is_set(): await asyncio.sleep(0.001) - return await super().async_increment(key, value) + return await super().async_increment(key, value, **kwargs) redis: Final = _ThreadHeldRedis() _process(monkeypatch, redis, cached_spend=6.0) @@ -2326,12 +2332,12 @@ class _DroppedFirstIncrementRedis(_SharedRedis): super().__init__() self.dropped = False - async def async_increment(self, key: str, value: float, **_: object) -> float: + async def async_increment(self, key: str, value: float, **kwargs: bool) -> float: if not self.dropped: self.dropped = True await asyncio.sleep(0) raise ConnectionError("connection reset by peer") - return await super().async_increment(key, value) + return await super().async_increment(key, value, **kwargs) @pytest.mark.asyncio @@ -2357,16 +2363,18 @@ class _StalledFirstIncrementRedis(_SharedRedis): self.stalled: Final = asyncio.Event() self.release: Final = asyncio.Event() - async def async_increment(self, key: str, value: float, **_: object) -> float: + async def async_increment(self, key: str, value: float, **kwargs: bool) -> float: if not self.stalled.is_set(): self.stalled.set() await self.release.wait() raise TimeoutError("redis did not answer") - return await super().async_increment(key, value) + return await super().async_increment(key, value, **kwargs) @pytest.mark.asyncio -async def test_cold_starts_stop_waiting_on_a_stalled_seed_and_add_their_own(monkeypatch: pytest.MonkeyPatch) -> None: +async def test_cold_starts_stop_waiting_on_a_stalled_seed_and_set_the_missing_counter_once( + monkeypatch: pytest.MonkeyPatch, +) -> None: counter: Final = "spend:end_user:stalled-seed" redis: Final = _StalledFirstIncrementRedis() _process(monkeypatch, redis, cached_spend=6.0) @@ -2381,7 +2389,7 @@ async def test_cold_starts_stop_waiting_on_a_stalled_seed_and_add_their_own(monk with pytest.raises(TimeoutError): await stalled - assert seeded_while_stalled == 18.0 + assert seeded_while_stalled == 6.0 class _FakeRedisHoldingIncrements(FakeAsyncRedis): @@ -2405,11 +2413,22 @@ async def test_a_cold_start_after_the_redis_breaker_opened_fails_fast_instead_of client: Final = _FakeRedisHoldingIncrements(FakeServer()) redis: Final = _RedisCacheOnClient(client, breaker) _process(monkeypatch, redis, cached_spend=6.0) + read: Final = asyncio.Event() + opened: Final = asyncio.Event() + + async def _read_before_the_seed_started() -> None: + with spend_counter_batch_scope(redis, frozenset((counter,))): + await _read_cold(counter) + read.set() + await opened.wait() + await ps.ensure_spend_counter_initialized(counter_key=counter, source_cache_key=_COLD_SEED_SOURCE) + + later: Final = asyncio.ensure_future(_read_before_the_seed_started()) + await read.wait() first: Final = asyncio.ensure_future(_cold_start(redis, counter)) await client.holding.wait() breaker.record_failure() - - later: Final = asyncio.ensure_future(_cold_start(redis, counter)) + opened.set() for _ in range(100): await asyncio.sleep(0) client.release.set() @@ -2453,64 +2472,6 @@ async def test_a_seed_that_lands_while_a_cold_read_is_on_its_way_back_is_not_add assert redis.values[counter] == 6.0 -class _HeldReadRedis(_SharedRedis): - """Answers a GET with the value its key had when it arrived, holding the reply while ``hold_reads`` is set.""" - - def __init__(self) -> None: - super().__init__() - self.hold_reads = False - self.read_arrived: Final = asyncio.Event() - self.release_read: Final = asyncio.Event() - self.reads_after_hold = 0 - - async def async_batch_get_cache(self, key_list: list[str], **_: object) -> dict[str, float | None]: - self.reads_after_hold += self.hold_reads - return await super().async_batch_get_cache(key_list) - - async def async_get_cache(self, key: str, **_: object) -> float | None: - self.reads_after_hold += self.hold_reads - value: Final = self.values.get(key) - if self.hold_reads: - self.read_arrived.set() - await self.release_read.wait() - return value - - -@pytest.mark.asyncio -async def test_a_seed_that_lands_while_a_request_re_reads_the_counter_is_not_added_again( - monkeypatch: pytest.MonkeyPatch, -) -> None: - counter: Final = "spend:end_user:seed-during-reread" - redis: Final = _HeldReadRedis() - _process(monkeypatch, redis, cached_spend=6.0) - read: Final = asyncio.Event() - seeded: Final = asyncio.Event() - - async def _stale_request() -> None: - with spend_counter_batch_scope(cast(RedisCache, redis), frozenset((counter,))): - await _read_cold(counter) - read.set() - await seeded.wait() - del redis.values[counter] - redis.hold_reads = True - await ps.ensure_spend_counter_initialized(counter_key=counter, source_cache_key=_COLD_SEED_SOURCE) - - async def _first_request() -> None: - await read.wait() - await _cold_start(redis, counter) - seeded.set() - - async def _request_seeding_during_the_re_read() -> None: - await redis.read_arrived.wait() - redis.hold_reads = False - await _cold_start(redis, counter) - redis.release_read.set() - - await asyncio.gather(_stale_request(), _first_request(), _request_seeding_during_the_re_read()) - - assert redis.values[counter] == 6.0 - - class _HeldGetRedis(_SharedRedis): """Answers the first GET with the value its key had when it arrived, holding the reply until released.""" @@ -2598,11 +2559,11 @@ async def test_a_cold_read_sent_before_an_add_started_does_not_join_it_when_the_ @pytest.mark.asyncio -async def test_a_re_read_redis_does_not_answer_stops_at_the_wait_budget_and_the_request_adds_its_cached_spend( +async def test_a_counter_lost_and_re_created_below_the_seed_gets_the_cached_spend_on_top( monkeypatch: pytest.MonkeyPatch, ) -> None: - counter: Final = "spend:end_user:stalled-reread" - redis: Final = _HeldReadRedis() + counter: Final = "spend:end_user:re-created-below" + redis: Final = _SharedRedis() _process(monkeypatch, redis, cached_spend=6.0) read: Final = asyncio.Event() seeded: Final = asyncio.Event() @@ -2612,8 +2573,36 @@ async def test_a_re_read_redis_does_not_answer_stops_at_the_wait_budget_and_the_ await _read_cold(counter) read.set() await seeded.wait() - del redis.values[counter] - redis.hold_reads = True + redis.values[counter] = 0.5 + await ps.ensure_spend_counter_initialized(counter_key=counter, source_cache_key=_COLD_SEED_SOURCE) + assert await read_batched_spend_counter(counter) == (6.5, True) + + async def _first_request() -> None: + await read.wait() + await _cold_start(redis, counter) + seeded.set() + + await asyncio.gather(_stale_request(), _first_request()) + + assert redis.values[counter] == 6.5 + + +@pytest.mark.asyncio +async def test_a_counter_holding_the_seed_is_left_alone_by_a_request_whose_source_cache_moved_past_it( + monkeypatch: pytest.MonkeyPatch, +) -> None: + counter: Final = "spend:end_user:source-moved" + redis: Final = _SharedRedis() + counters: Final = _process(monkeypatch, redis, cached_spend=6.0) + read: Final = asyncio.Event() + seeded: Final = asyncio.Event() + + async def _stale_request() -> None: + with spend_counter_batch_scope(cast(RedisCache, redis), frozenset((counter,))): + await _read_cold(counter) + read.set() + await seeded.wait() + ps.user_api_key_cache.in_memory_cache.set_cache(key=_COLD_SEED_SOURCE, value={"spend": 6.25}) await ps.ensure_spend_counter_initialized(counter_key=counter, source_cache_key=_COLD_SEED_SOURCE) async def _first_request() -> None: @@ -2621,10 +2610,153 @@ async def test_a_re_read_redis_does_not_answer_stops_at_the_wait_budget_and_the_ await _cold_start(redis, counter) seeded.set() - await asyncio.wait_for(asyncio.gather(_stale_request(), _first_request()), timeout=2.0) + await asyncio.gather(_stale_request(), _first_request()) assert redis.values[counter] == 6.0 - assert redis.reads_after_hold == 1 + assert counters.in_memory_cache.get_cache(key=counter) == 6.0 + + +@pytest.mark.asyncio +async def test_a_request_done_waiting_on_a_stalled_seed_adds_when_the_counter_holds_less_than_its_cached_spend( + monkeypatch: pytest.MonkeyPatch, +) -> None: + counter: Final = "spend:end_user:below-own-spend" + monkeypatch.setattr(spend_counter_reseed, "SPEND_COUNTER_COLD_SEED_JOIN_SECONDS", 0.01) + redis: Final = _HeldIncrementRedis(held_increments=1) + _process(monkeypatch, redis, cached_spend=6.0) + stalled: Final = asyncio.ensure_future(_cold_start(redis, counter)) + await redis.arrived[0].wait() + + with spend_counter_batch_scope(cast(RedisCache, redis), frozenset((counter,))): + await _read_cold(counter) + redis.values[counter] = 0.5 + await ps.ensure_spend_counter_initialized(counter_key=counter, source_cache_key=_COLD_SEED_SOURCE) + while_stalled: Final = redis.values[counter] + redis.releases[0].set() + await stalled + + assert while_stalled == 6.5 + + +@pytest.mark.asyncio +async def test_a_counter_without_redis_set_only_if_missing_keeps_the_value_it_holds( + monkeypatch: pytest.MonkeyPatch, +) -> None: + counter: Final = "spend:end_user:no-redis-set-if-missing" + monkeypatch.setattr(ps, "spend_counter_cache", DualCache(in_memory_cache=InMemoryCache())) + + first: Final = await ps.increment_spend_counter_cache(counter, 6.0, only_if_missing=True) + second: Final = await ps.increment_spend_counter_cache(counter, 4.0, only_if_missing=True) + added: Final = await ps.increment_spend_counter_cache(counter, 1.0) + + assert (first, second, added) == (6.0, 6.0, 7.0) + + +@pytest.mark.asyncio +async def test_a_counter_set_by_a_request_done_waiting_counts_as_the_seed_for_a_later_request( + monkeypatch: pytest.MonkeyPatch, +) -> None: + counter: Final = "spend:end_user:set-counts-as-seed" + monkeypatch.setattr(spend_counter_reseed, "SPEND_COUNTER_COLD_SEED_JOIN_SECONDS", 0.01) + redis: Final = _HeldIncrementRedis(held_increments=1) + _process(monkeypatch, redis, cached_spend=6.0) + read: Final = asyncio.Event() + set_by_a_waiter: Final = asyncio.Event() + + async def _stale_request() -> None: + with spend_counter_batch_scope(cast(RedisCache, redis), frozenset((counter,))): + await _read_cold(counter) + read.set() + await set_by_a_waiter.wait() + redis.values[counter] = 0.5 + await ps.ensure_spend_counter_initialized(counter_key=counter, source_cache_key=_COLD_SEED_SOURCE) + + stale: Final = asyncio.ensure_future(_stale_request()) + await read.wait() + stalled: Final = asyncio.ensure_future(_cold_start(redis, counter)) + await redis.arrived[0].wait() + await _cold_start(redis, counter) + set_by_a_waiter.set() + await stale + after_the_stale_request: Final = redis.values[counter] + redis.releases[0].set() + await stalled + + assert after_the_stale_request == 6.5 + + +class _CommandCountingRedis(_SharedRedis): + def __init__(self) -> None: + super().__init__() + self.commands: Final[list[str]] = [] # mutable-ok: the commands the test asserts on + + async def async_batch_get_cache(self, key_list: list[str], **_: object) -> dict[str, float | None]: + self.commands.append("mget") + return await super().async_batch_get_cache(key_list) + + async def async_get_cache(self, key: str, **_: object) -> float | None: + self.commands.append("get") + return await super().async_get_cache(key) + + async def async_increment(self, key: str, value: float, **kwargs: bool) -> float: + self.commands.append("set_if_missing" if kwargs.get("only_if_missing") else "add") + return await super().async_increment(key, value, **kwargs) + + +@pytest.mark.asyncio +async def test_a_request_that_read_before_a_seed_landed_sends_one_write_and_keeps_its_value_for_later_reads( + monkeypatch: pytest.MonkeyPatch, +) -> None: + counter: Final = "spend:end_user:one-write" + redis: Final = _CommandCountingRedis() + _process(monkeypatch, redis, cached_spend=6.0) + read: Final = asyncio.Event() + seeded: Final = asyncio.Event() + + async def _stale_request() -> None: + with spend_counter_batch_scope(cast(RedisCache, redis), frozenset((counter,))): + await _read_cold(counter) + read.set() + await seeded.wait() + redis.commands.clear() + await ps.ensure_spend_counter_initialized(counter_key=counter, source_cache_key=_COLD_SEED_SOURCE) + assert await read_batched_spend_counter(counter) == (6.0, True) + assert await ps.get_current_spend(counter_key=counter, fallback_spend=6.0) == 6.0 + + async def _first_request() -> None: + await read.wait() + await _cold_start(redis, counter) + seeded.set() + + await asyncio.gather(_stale_request(), _first_request()) + + assert redis.commands == ["set_if_missing"] + + +@pytest.mark.asyncio +async def test_a_request_waits_on_other_requests_adds_for_the_limit_in_total_across_its_counters( + monkeypatch: pytest.MonkeyPatch, +) -> None: + limit: Final = 0.3 + monkeypatch.setattr(spend_counter_reseed, "SPEND_COUNTER_COLD_SEED_JOIN_SECONDS", limit) + keys: Final = tuple(f"spend:end_user:shared-wait-{index}" for index in range(4)) + redis: Final = _HeldIncrementRedis(held_increments=len(keys)) + _process(monkeypatch, redis, cached_spend=6.0) + stalled: Final = [asyncio.ensure_future(_cold_start(redis, key)) for key in keys] + for arrived in redis.arrived: + await arrived.wait() + + loop: Final = asyncio.get_running_loop() + started: Final = loop.time() + with spend_counter_batch_scope(cast(RedisCache, redis), frozenset(keys)): + for key in keys: + await ps.ensure_spend_counter_initialized(counter_key=key, source_cache_key=_COLD_SEED_SOURCE) + waited: Final = loop.time() - started + for release in redis.releases: + release.set() + await asyncio.gather(*stalled) + + assert limit <= waited < 2 * limit class _LostAndRecreatedAfterFailedMGetRedis(_SharedRedis): @@ -2657,9 +2789,9 @@ class _LostAndRecreatedAfterFailedMGetRedis(_SharedRedis): self.cold_gets += 1 return await super().async_get_cache(key) - async def async_increment(self, key: str, value: float, **_: object) -> float: + async def async_increment(self, key: str, value: float, **kwargs: bool) -> float: self._other_process_increments_after_the_cold_gets() - return await super().async_increment(key, value) + return await super().async_increment(key, value, **kwargs) @pytest.mark.asyncio @@ -2686,8 +2818,8 @@ class _AppliedHeldThenResetRedis(_HeldReplyRedis): super().__init__() self.deleted = False - async def async_increment(self, key: str, value: float, **_: object) -> float: - await super().async_increment(key, value) + async def async_increment(self, key: str, value: float, **kwargs: bool) -> float: + await super().async_increment(key, value, **kwargs) raise ConnectionError("connection reset") async def async_delete_cache(self, key: str) -> None: @@ -2721,6 +2853,7 @@ async def test_requests_waiting_on_a_cold_seed_whose_request_was_cancelled_add_t monkeypatch: pytest.MonkeyPatch, ) -> None: counter: Final = "spend:end_user:cancelled-seed-with-waiters" + monkeypatch.setattr(spend_counter_reseed, "SPEND_COUNTER_COLD_SEED_JOIN_SECONDS", 60.0) redis: Final = _StalledFirstIncrementRedis() _process(monkeypatch, redis, cached_spend=6.0) read: Final = asyncio.Event() @@ -2742,7 +2875,7 @@ async def test_requests_waiting_on_a_cold_seed_whose_request_was_cancelled_add_t for _ in range(20): await asyncio.sleep(0) seed.cancel() - await asyncio.gather(joined_before_start, joined_after_start) + await asyncio.wait_for(asyncio.gather(joined_before_start, joined_after_start), timeout=5) assert seed.cancelled() assert redis.values[counter] == 6.0 @@ -2766,13 +2899,13 @@ class _HeldIncrementRedis(_SharedRedis): self.reads += 1 return await super().async_get_cache(key) - async def async_increment(self, key: str, value: float, **_: object) -> float: + async def async_increment(self, key: str, value: float, **kwargs: bool) -> float: index: Final = self.increments self.increments += 1 if index < len(self.arrived): self.arrived[index].set() await self.releases[index].wait() - return await super().async_increment(key, value) + return await super().async_increment(key, value, **kwargs) @pytest.mark.asyncio @@ -2830,23 +2963,11 @@ async def test_a_request_that_read_the_counter_while_a_seed_was_in_flight_does_n assert redis.values[counter] == 6.0 -class _SlowGetRedis(_HeldIncrementRedis): - """Also takes ``SECONDS`` to answer a GET.""" - - SECONDS: ClassVar[float] = 0.2 - - async def async_get_cache(self, key: str, **_: object) -> float | None: - await asyncio.sleep(self.SECONDS) - return await super().async_get_cache(key) - - @pytest.mark.asyncio -async def test_a_seed_that_lands_late_in_the_wait_leaves_time_to_re_read_the_counter_instead_of_adding_again( - monkeypatch: pytest.MonkeyPatch, -) -> None: +async def test_a_seed_that_lands_late_in_the_wait_is_not_added_again(monkeypatch: pytest.MonkeyPatch) -> None: counter: Final = "spend:end_user:late-landing-seed" monkeypatch.setattr(spend_counter_reseed, "SPEND_COUNTER_COLD_SEED_JOIN_SECONDS", 0.5) - redis: Final = _SlowGetRedis(held_increments=1) + redis: Final = _HeldIncrementRedis(held_increments=1) _process(monkeypatch, redis, cached_spend=6.0) first: Final = asyncio.ensure_future(_cold_start(redis, counter)) @@ -2859,6 +2980,35 @@ async def test_a_seed_that_lands_late_in_the_wait_leaves_time_to_re_read_the_cou assert redis.values[counter] == 6.0 +@pytest.mark.asyncio +async def test_a_request_joining_an_add_that_started_after_its_read_came_back_sends_no_write( + monkeypatch: pytest.MonkeyPatch, +) -> None: + counter: Final = "spend:end_user:joined-no-write" + monkeypatch.setattr(spend_counter_reseed, "SPEND_COUNTER_COLD_SEED_JOIN_SECONDS", 60.0) + redis: Final = _HeldIncrementRedis(held_increments=1) + _process(monkeypatch, redis, cached_spend=6.0) + read: Final = asyncio.Event() + + async def _joining_request() -> None: + with spend_counter_batch_scope(cast(RedisCache, redis), frozenset((counter,))): + await _read_cold(counter) + read.set() + await redis.arrived[0].wait() + await ps.ensure_spend_counter_initialized(counter_key=counter, source_cache_key=_COLD_SEED_SOURCE) + + joining: Final = asyncio.ensure_future(_joining_request()) + await read.wait() + first: Final = asyncio.ensure_future(_cold_start(redis, counter)) + await redis.arrived[0].wait() + for _ in range(100): + await asyncio.sleep(0) + redis.releases[0].set() + await asyncio.gather(joining, first) + + assert (redis.values[counter], redis.increments) == (6.0, 1) + + class _HeldSecondGetRedis(_HeldIncrementRedis): """Also answers the second GET with the value its key had when it arrived, holding the reply until released.""" @@ -2917,7 +3067,63 @@ async def test_a_stalled_seed_finishing_does_not_hide_the_newer_seed_still_in_fl late: Final = asyncio.ensure_future(_cold_start(redis, counter)) for _ in range(10): await asyncio.sleep(0) + commands_while_the_newer_seed_was_in_flight: Final = redis.increments redis.releases[1].set() await asyncio.gather(newer, late) + assert commands_while_the_newer_seed_was_in_flight == 2 assert redis.values[counter] == 6.0 + + +class _TargetRecordingRedis(_SharedRedis): + def __init__(self) -> None: + super().__init__() + self.targets: Final[list[str | None]] = [] # mutable-ok: the targets the test asserts on + + async def async_get_cache(self, key: str, **_: object) -> float | None: + self.targets.append(current_service_target()) + return 1.0 + + +@pytest.mark.asyncio +async def test_the_cold_check_read_reports_under_spend_counters_inside_another_phases_target( + monkeypatch: pytest.MonkeyPatch, +) -> None: + redis: Final = _TargetRecordingRedis() + _process(monkeypatch, redis, cached_spend=6.0) + + with service_target(AUTH_OBJECTS_TARGET): + await ps.ensure_spend_counter_initialized(counter_key="spend:key:target", source_cache_key=_COLD_SEED_SOURCE) + + assert redis.targets == [SPEND_COUNTERS_TARGET] + + +@pytest.mark.asyncio +async def test_a_stale_read_on_a_real_client_seeds_a_missing_counter_with_its_ttl_and_keeps_an_existing_one( + monkeypatch: pytest.MonkeyPatch, +) -> None: + counter: Final = "spend:end_user:real-client" + client: Final = FakeAsyncRedis(server=FakeServer()) + redis: Final = _RedisCacheOnClient(client) + redis.default_ttl = 120 + _process(monkeypatch, redis, cached_spend=6.0) + read: Final = asyncio.Event() + seeded: Final = asyncio.Event() + + async def _stale_request() -> None: + with spend_counter_batch_scope(redis, frozenset((counter,))): + await _read_cold(counter) + read.set() + await seeded.wait() + await client.delete(counter) + await ps.ensure_spend_counter_initialized(counter_key=counter, source_cache_key=_COLD_SEED_SOURCE) + + async def _first_request() -> None: + await read.wait() + await _cold_start(redis, counter) + seeded.set() + + await asyncio.gather(_stale_request(), _first_request()) + + assert float(await client.get(counter) or 0) == 6.0 + assert 0 < await client.ttl(counter) <= 120