diff --git a/litellm/caching/redis_cache.py b/litellm/caching/redis_cache.py index 3b90cce211d..1e469e26990 100644 --- a/litellm/caching/redis_cache.py +++ b/litellm/caching/redis_cache.py @@ -1473,15 +1473,29 @@ class RedisCache(BaseCache): await _redis_client.expire(key, ttl) return float(result) + async def _set_if_missing_with_ttl(self, key: str, value: float, ttl: int | None) -> float: + """SET NX, its TTL command and a GET in one round trip; the value the key holds afterwards.""" + async with self._async_commands().pipeline(transaction=False) as pipe: + pipe.set(name=key, value=value, nx=True) + if ttl is not None: + pipe.expire(key, ttl) + pipe.get(key) + replies: Final[Sequence[object]] = await pipe.execute() # pyright: ignore[reportUnknownVariableType, reportUnknownMemberType] # redis generics + current: Final = _LUA_OPTIONAL_TOTAL.validate_python(replies[-1]) + return value if current is None else current + @_redis_circuit_breaker_guard async def async_increment( self, - key, + key: str, value: float, ttl: int | None = None, parent_otel_span: Span | None = None, refresh_ttl: bool = False, + only_if_missing: bool = False, ) -> float: + """Add ``value`` to the key and return its new value. With ``only_if_missing``, a key that exists keeps its + value, which is returned, and only the TTL is refreshed.""" from redis.asyncio import Redis _redis_client: Final[Redis] = self.init_async_client() @@ -1489,8 +1503,12 @@ class RedisCache(BaseCache): _used_ttl: Final = self.get_ttl(ttl=ttl) key = self.check_and_fix_namespace(key=key) try: - result: Final = await self._incrbyfloat_with_ttl( - _redis_client, key=key, value=value, ttl=_used_ttl, refresh_ttl=refresh_ttl + result: Final = await ( + self._set_if_missing_with_ttl(key=key, value=value, ttl=_used_ttl) + if only_if_missing + else self._incrbyfloat_with_ttl( + _redis_client, key=key, value=value, ttl=_used_ttl, refresh_ttl=refresh_ttl + ) ) ## LOGGING ## diff --git a/litellm/proxy/db/spend_counter_reseed.py b/litellm/proxy/db/spend_counter_reseed.py index f47dc17bbd6..8a0d413bdbc 100644 --- a/litellm/proxy/db/spend_counter_reseed.py +++ b/litellm/proxy/db/spend_counter_reseed.py @@ -16,6 +16,7 @@ import asyncio import functools from collections import OrderedDict from collections.abc import Awaitable, Callable, Mapping +from contextvars import ContextVar from dataclasses import dataclass from datetime import datetime, timezone from types import MappingProxyType @@ -30,7 +31,6 @@ from litellm.proxy.db.db_lookup_gate import bounded_db_lookup, db_lookup_gate from litellm.proxy.spend_tracking.spend_counter_batch import ( SPEND_COUNTERS_TARGET, SpendCounterRead, - forget_spend_counter, read_batched_spend_counter, record_spend_counter_value, spend_counter_tick, @@ -75,11 +75,12 @@ _WINDOW_SPEND_LOG_FIELDS: Final[Mapping[str, str]] = MappingProxyType( @dataclass(slots=True) class _CachedSpendSeed: - """This process adding its cached spend to one cold counter: the add in flight, the tick it started at, and the - tick the last one landed at.""" + """This process adding its cached spend to one cold counter: the add in flight, the tick it started at, the tick + the last one landed at, and the least the counter held when that one landed.""" started_at: int = 0 landed_at: int = 0 + landed_spend: float = 0.0 in_flight: asyncio.Future[float | None] | None = None def on_this_loop(self) -> bool: @@ -107,6 +108,18 @@ def _cached_spend_seed(spend_counter_cache: "DualCache", counter_key: str) -> _C return _CachedSpendSeed() +_cold_seed_waited: Final[ContextVar[float]] = ContextVar("cold_spend_seed_waited", default=0.0) + + +async def _wait_for_cold_seed(in_flight: asyncio.Future[float | None]) -> None: + """Wait on ``in_flight`` for what this context has left of ``SPEND_COUNTER_COLD_SEED_JOIN_SECONDS``.""" + loop: Final = asyncio.get_running_loop() + waited: Final = _cold_seed_waited.get() + started: Final = loop.time() + await asyncio.wait((in_flight,), timeout=max(0.0, SPEND_COUNTER_COLD_SEED_JOIN_SECONDS - waited)) + _cold_seed_waited.set(waited + loop.time() - started) + + def _redis_breaker_closed(spend_counter_cache: "DualCache") -> bool: redis_cache: Final = spend_counter_cache.redis_cache return redis_cache is None or redis_cache.circuit_breaker_closed @@ -161,56 +174,66 @@ class SpendCounterReseed: spend_counter_cache: "DualCache", counter_key: str, read: SpendCounterRead, + spend: float, add: Callable[[], Awaitable[float | None]], - is_warm: Callable[[], Awaitable[bool]], + set_if_missing: Callable[[], Awaitable[float | None]], ) -> None: - """Run ``add``, this process adding its cached spend to a cold counter, once for all of this process's - requests that read the counter cold before that add landed. A request joins an add that started after its read - came back and gets its result, and starts or joins another add when that one fails or is cancelled. When an add - landed after its read went out, or one that started before its read came back finishes, the request re-reads - the counter and adds again only if the counter is gone. A request waits on other requests' adds for at most - ``SPEND_COUNTER_COLD_SEED_JOIN_SECONDS`` in all, has that long again to finish a re-read, and then adds its own. - A request on another event loop than the add in flight, one that finds the Redis circuit breaker not closed, and - each other process, still add their own cached spend.""" + """Run ``add``, this process adding ``spend`` to a cold counter, once for all of this process's requests that + read the counter cold before that add landed. A request joins an add that started after its read came back and + gets its result, and starts or joins another add when that one fails or is cancelled. A request whose read went + out before an add landed, and one still waiting on an add after ``SPEND_COUNTER_COLD_SEED_JOIN_SECONDS`` across + its counters, runs ``set_if_missing`` in place of ``add``, then runs ``add`` too if the counter holds less than + the spend it should already hold. A request on another event loop than the add in flight, one that finds the + Redis circuit breaker not closed, and each other process, still run ``add``.""" seed: Final = _cached_spend_seed(spend_counter_cache, counter_key) - loop: Final = asyncio.get_running_loop() - join_by: Final = loop.time() + SPEND_COUNTER_COLD_SEED_JOIN_SECONDS - re_read_by: Final = join_by + SPEND_COUNTER_COLD_SEED_JOIN_SECONDS - async def add_unless_covered(cold: SpendCounterRead) -> float | None: + async def add_spend() -> float | None: + added: Final = await add() + seed.landed_spend = spend + return added + + async def add_if_short() -> float | None: + least: Final = min(spend, seed.landed_spend) if seed.landed_at >= read.sent_at else spend + current: Final = await set_if_missing() + if current is not None and current < least: + return await add_spend() + seed.landed_spend = least + return current + + async def add_unless_covered() -> float | None: in_flight: Final = seed.in_flight if not seed.on_this_loop() or not _redis_breaker_closed(spend_counter_cache): return await add() if in_flight is None: - if seed.landed_at < cold.sent_at: - return await seed.run(add) - forget_spend_counter(counter_key) - sent_at: Final = spend_counter_tick() - try: - warm: Final = await asyncio.wait_for(is_warm(), timeout=max(0.0, re_read_by - loop.time())) - except asyncio.TimeoutError: - return await seed.run(add) - if warm: - return None - return await add_unless_covered(SpendCounterRead(sent_at=sent_at, returned_at=spend_counter_tick())) - covers: Final = seed.started_at > cold.returned_at - await asyncio.wait((in_flight,), timeout=max(0.0, join_by - loop.time())) + return await seed.run(add_spend if seed.landed_at < read.sent_at else add_if_short) + covers: Final = seed.started_at > read.returned_at + await _wait_for_cold_seed(in_flight) if not in_flight.done(): - return await seed.run(add) + return await seed.run(add_if_short) if covers and not in_flight.cancelled(): return in_flight.result() - return await add_unless_covered(cold) + return await add_unless_covered() - added: Final = await add_unless_covered(read) + added: Final = await add_unless_covered() if added is not None: record_spend_counter_value(counter_key, added) @staticmethod @with_service_target(SPEND_COUNTERS_TARGET) - async def increment_in_memory(spend_counter_cache: "DualCache", counter_key: str, increment: float) -> float | None: - """Apply local deltas after an in-flight reseed establishes the spend balance.""" + async def increment_in_memory( + spend_counter_cache: "DualCache", counter_key: str, increment: float, only_if_missing: bool = False + ) -> float | None: + """Apply local deltas after an in-flight reseed establishes the spend balance. With ``only_if_missing``, a + counter that exists keeps its value, which is returned.""" lock: Final = await SpendCounterReseed._get_lock(counter_key) async with lock: + current: Final[float | None] = ( # pyright: ignore[reportUnknownVariableType] # untyped in-memory cache + spend_counter_cache.in_memory_cache.get_cache(key=counter_key) # pyright: ignore[reportUnknownMemberType] # untyped in-memory cache + if only_if_missing + else None + ) + if current is not None: + return float(current) return await spend_counter_cache.async_increment_cache( key=counter_key, value=increment, local_only=True, refresh_ttl=True ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 741f7d58f1d..7c1676e22c1 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -930,7 +930,6 @@ from litellm.proxy.spend_tracking.spend_capture_rate import ( from litellm.proxy.spend_tracking.spend_counter_batch import ( SPEND_COUNTERS_TARGET, PendingSpendIncrement, - SpendCounterRead, active_spend_counter_batch, forget_spend_counter, post_call_counter_keys, @@ -4087,8 +4086,9 @@ async def ensure_spend_counter_initialized( counter_key: str, source_cache_key: str | list[str], ) -> None: - read: Final = await _cold_spend_counter_read(counter_key) - if read is not None: + is_warm: Final = await _is_spend_counter_cache_warm(counter_key=counter_key) + if is_warm is False: + read: Final = spend_counter_read(counter_key) # Shares the per-counter lock with get_current_spend. db_spend: Final = await SpendCounterReseed.coalesced( prisma_client=prisma_client, @@ -4104,8 +4104,11 @@ async def ensure_spend_counter_initialized( spend_counter_cache=spend_counter_cache, counter_key=counter_key, read=read, + spend=base_spend, add=lambda: increment_spend_counter_cache(counter_key=counter_key, increment=base_spend), - is_warm=lambda: _is_spend_counter_warm_outside_batch(counter_key), + set_if_missing=lambda: increment_spend_counter_cache( + counter_key=counter_key, increment=base_spend, only_if_missing=True + ), ) @@ -4160,22 +4163,14 @@ _ensure_window_spend_counter_initialized: Final = ensure_window_spend_counter_in @with_service_target(SPEND_COUNTERS_TARGET) async def _is_spend_counter_cache_warm(counter_key: str) -> bool: - return await _cold_spend_counter_read(counter_key) is None - - -async def _cold_spend_counter_read(counter_key: str) -> SpendCounterRead | None: - """When the read that found the counter missing went out and came back, None when the counter has a value. A read - outside the batch counts as now, since ``SpendCounterReseed.coalesced`` reads the counter again after it.""" batched: Final = await read_batched_spend_counter(counter_key) - if batched is not None and batched[0] is not None: - spend_counter_cache.in_memory_cache.set_cache(key=counter_key, value=batched[0]) - return None - if batched is None and await _is_spend_counter_warm_outside_batch(counter_key): - return None - return spend_counter_read(counter_key) + if batched is not None: + batched_value, _ = batched + if batched_value is None: + return False + spend_counter_cache.in_memory_cache.set_cache(key=counter_key, value=batched_value) + return True - -async def _is_spend_counter_warm_outside_batch(counter_key: str) -> bool: if spend_counter_cache.redis_cache is not None: try: current_value: Final[object] = await spend_counter_cache.redis_cache.async_get_cache( @@ -4217,13 +4212,24 @@ async def refresh_spend_counter_ttl(counter_key: str) -> bool: @with_service_target(SPEND_COUNTERS_TARGET) -async def increment_spend_counter_cache(counter_key: str, increment: float) -> float | None: +async def increment_spend_counter_cache( + counter_key: str, increment: float, only_if_missing: bool = False +) -> float | None: if spend_counter_cache.redis_cache is not None: try: - current_value: Final = await spend_counter_cache.redis_cache.async_increment( - key=counter_key, - value=increment, - refresh_ttl=True, + current_value: Final = await ( + spend_counter_cache.redis_cache.async_increment( + key=counter_key, + value=increment, + refresh_ttl=True, + only_if_missing=True, + ) + if only_if_missing + else spend_counter_cache.redis_cache.async_increment( + key=counter_key, + value=increment, + refresh_ttl=True, + ) ) except Exception: await invalidate_spend_counter(counter_key=counter_key) @@ -4236,7 +4242,10 @@ async def increment_spend_counter_cache(counter_key: str, increment: float) -> f return current_value return await SpendCounterReseed.increment_in_memory( - spend_counter_cache=spend_counter_cache, counter_key=counter_key, increment=increment + spend_counter_cache=spend_counter_cache, + counter_key=counter_key, + increment=increment, + only_if_missing=only_if_missing, ) diff --git a/litellm/proxy/spend_tracking/spend_counter_batch.py b/litellm/proxy/spend_tracking/spend_counter_batch.py index 55c4fab6893..8ff709086bb 100644 --- a/litellm/proxy/spend_tracking/spend_counter_batch.py +++ b/litellm/proxy/spend_tracking/spend_counter_batch.py @@ -188,7 +188,8 @@ def active_spend_counter_batch() -> SpendCounterBatch | None: def spend_counter_read(counter_key: str) -> SpendCounterRead: - """When this context's read of the counter went out and came back: the open batch's MGET for it, else now.""" + """When this context's read of the counter went out and came back: the open batch's MGET for it, else now, since + ``SpendCounterReseed.coalesced`` reads the counter again after a read outside the batch.""" batch: Final = _active_batch.get() batched: Final = batch.counter_read(counter_key) if batch is not None else None if batched is not None: diff --git a/tests/unit/caching/test_redis_cache.py b/tests/unit/caching/test_redis_cache.py index 2bdc7b9549b..356a1f34985 100644 --- a/tests/unit/caching/test_redis_cache.py +++ b/tests/unit/caching/test_redis_cache.py @@ -8,6 +8,7 @@ from typing import Final, cast from unittest.mock import AsyncMock, MagicMock, patch import pytest +from fakeredis import FakeAsyncRedis, FakeServer import litellm from litellm._service_logger import ServiceLogging @@ -1474,6 +1475,55 @@ async def test_async_increment_pipelines_the_ttl_command( assert client.round_trips == expected_round_trips +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("existing", "expected"), + [ + pytest.param(None, 1.5, id="missing key: set to the value"), + pytest.param(0.25, 0.25, id="key below the value: kept"), + pytest.param(4.0, 4.0, id="key above the value: kept"), + ], +) +async def test_async_increment_only_if_missing_sets_a_missing_key_and_refreshes_the_ttl( + monkeypatch, redis_no_ping, existing, expected +): + monkeypatch.setenv("REDIS_HOST", "https://my-test-host") + redis_cache = RedisCache(namespace="ns") + client = FakeAsyncRedis(server=FakeServer()) + if existing is not None: + await client.set("ns:spend:key:k", existing, ex=5) + + with patch.object(redis_cache, "init_async_client", return_value=client): + current = await redis_cache.async_increment( + key="spend:key:k", value=1.5, ttl=60, refresh_ttl=True, only_if_missing=True + ) + after_add = await redis_cache.async_increment(key="spend:key:k", value=2.0, ttl=60, refresh_ttl=True) + + assert current == expected + assert after_add == expected + 2.0 + assert await client.ttl("ns:spend:key:k") == 60 + + +@pytest.mark.asyncio +async def test_async_increment_only_if_missing_reports_a_redis_failure_like_an_increment(monkeypatch, redis_no_ping): + monkeypatch.setenv("REDIS_HOST", "https://my-test-host") + redis_cache = RedisCache(namespace="ns") + client = MagicMock() + client.pipeline.side_effect = ConnectionError("connection reset") + failure_hook = AsyncMock() + + with ( + patch.object(redis_cache, "init_async_client", return_value=client), + patch.object(redis_cache.service_logger_obj, "async_service_failure_hook", failure_hook), + pytest.raises(ConnectionError), + ): + await redis_cache.async_increment(key="spend:key:k", value=1.5, ttl=60, refresh_ttl=True, only_if_missing=True) + await asyncio.sleep(0) + + assert failure_hook.await_args.kwargs["call_type"] == "async_increment" + assert redis_cache._circuit_breaker._hard_failure_count == 1 + + class _SetRecordingPipeline: def __init__(self) -> None: self.sets: list[tuple[str, str, timedelta | None]] = [] diff --git a/tests/unit/proxy/proxy_server/test_spend_counters.py b/tests/unit/proxy/proxy_server/test_spend_counters.py index c9342a1b895..e4a37721bba 100644 --- a/tests/unit/proxy/proxy_server/test_spend_counters.py +++ b/tests/unit/proxy/proxy_server/test_spend_counters.py @@ -25,7 +25,7 @@ import asyncio import threading from collections.abc import Awaitable, Callable from datetime import datetime -from typing import ClassVar, Final, Literal, cast +from typing import Final, Literal, cast from unittest.mock import AsyncMock, MagicMock, call import pytest @@ -37,13 +37,13 @@ from opentelemetry.trace import StatusCode import litellm import litellm.proxy.proxy_server as ps -from litellm._internal_context import current_service_target +from litellm._internal_context import current_service_target, service_target from litellm._service_logger import ServiceLogging from litellm.caching.dual_cache import DualCache from litellm.caching.in_memory_cache import InMemoryCache from litellm.caching.redis_batch import request_redis_batch_scope from litellm.caching.redis_cache import RedisCache, RedisCircuitBreaker, RedisCircuitBreakerOpenError -from litellm.constants import GLOBAL_PROXY_SPEND_CACHE_KEY +from litellm.constants import AUTH_OBJECTS_TARGET, GLOBAL_PROXY_SPEND_CACHE_KEY from litellm.integrations.opentelemetry import OpenTelemetry, OpenTelemetryConfig from litellm.models.end_user import LiteLLM_EndUserTable from litellm.models.tag import LiteLLM_TagTable @@ -55,6 +55,7 @@ from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache, end_u from litellm.proxy.db import spend_counter_reseed from litellm.proxy.management_endpoints.team_endpoints import refresh_cached_team from litellm.proxy.spend_tracking.spend_counter_batch import ( + SPEND_COUNTERS_TARGET, active_spend_counter_batch, read_batched_spend_counter, spend_counter_batch_scope, @@ -2044,12 +2045,16 @@ class _SharedRedis: await asyncio.sleep(0) return self.values.get(key) - async def async_increment(self, key: str, value: float, **_: object) -> float: + def _apply(self, key: str, value: float, only_if_missing: bool = False, **_: object) -> float: + if not only_if_missing or key not in self.values: + self.values[key] = self.values.get(key, 0.0) + value + return self.values[key] + + async def async_increment(self, key: str, value: float, **kwargs: bool) -> float: await asyncio.sleep(0) if self.failing: raise ConnectionError("redis down") - self.values[key] = self.values.get(key, 0.0) + value - return self.values[key] + return self._apply(key, value, **kwargs) async def async_delete_cache(self, key: str) -> None: await asyncio.sleep(0) @@ -2091,7 +2096,9 @@ async def test_overlapping_cold_starts_in_one_process_add_its_cached_spend_once( await asyncio.gather(*(_cold_start(redis, counter) for _ in range(5))) - seeded: Final = redis.values.get(counter) if redis is not None else counters.in_memory_cache.get_cache(key=counter) + seeded: Final[float | None] = ( + redis.values.get(counter) if redis is not None else counters.in_memory_cache.get_cache(key=counter) + ) assert seeded == 6.0 @@ -2205,9 +2212,8 @@ class _HeldReplyRedis(_SharedRedis): self.applied: Final = asyncio.Event() self.release: Final = asyncio.Event() - async def async_increment(self, key: str, value: float, **_: object) -> float: - self.values[key] = self.values.get(key, 0.0) + value - total: Final = self.values[key] + async def async_increment(self, key: str, value: float, **kwargs: bool) -> float: + total: Final = self._apply(key, value, **kwargs) self.applied.set() await self.release.wait() return total @@ -2240,12 +2246,12 @@ def test_a_cold_start_on_another_event_loop_adds_on_its_own(monkeypatch: pytest. release: Final = threading.Event() class _ThreadHeldRedis(_SharedRedis): - async def async_increment(self, key: str, value: float, **_: object) -> float: + async def async_increment(self, key: str, value: float, **kwargs: bool) -> float: if threading.current_thread() is not threading.main_thread(): held.set() while not release.is_set(): await asyncio.sleep(0.001) - return await super().async_increment(key, value) + return await super().async_increment(key, value, **kwargs) redis: Final = _ThreadHeldRedis() _process(monkeypatch, redis, cached_spend=6.0) @@ -2326,12 +2332,12 @@ class _DroppedFirstIncrementRedis(_SharedRedis): super().__init__() self.dropped = False - async def async_increment(self, key: str, value: float, **_: object) -> float: + async def async_increment(self, key: str, value: float, **kwargs: bool) -> float: if not self.dropped: self.dropped = True await asyncio.sleep(0) raise ConnectionError("connection reset by peer") - return await super().async_increment(key, value) + return await super().async_increment(key, value, **kwargs) @pytest.mark.asyncio @@ -2357,16 +2363,18 @@ class _StalledFirstIncrementRedis(_SharedRedis): self.stalled: Final = asyncio.Event() self.release: Final = asyncio.Event() - async def async_increment(self, key: str, value: float, **_: object) -> float: + async def async_increment(self, key: str, value: float, **kwargs: bool) -> float: if not self.stalled.is_set(): self.stalled.set() await self.release.wait() raise TimeoutError("redis did not answer") - return await super().async_increment(key, value) + return await super().async_increment(key, value, **kwargs) @pytest.mark.asyncio -async def test_cold_starts_stop_waiting_on_a_stalled_seed_and_add_their_own(monkeypatch: pytest.MonkeyPatch) -> None: +async def test_cold_starts_stop_waiting_on_a_stalled_seed_and_set_the_missing_counter_once( + monkeypatch: pytest.MonkeyPatch, +) -> None: counter: Final = "spend:end_user:stalled-seed" redis: Final = _StalledFirstIncrementRedis() _process(monkeypatch, redis, cached_spend=6.0) @@ -2381,7 +2389,7 @@ async def test_cold_starts_stop_waiting_on_a_stalled_seed_and_add_their_own(monk with pytest.raises(TimeoutError): await stalled - assert seeded_while_stalled == 18.0 + assert seeded_while_stalled == 6.0 class _FakeRedisHoldingIncrements(FakeAsyncRedis): @@ -2405,11 +2413,22 @@ async def test_a_cold_start_after_the_redis_breaker_opened_fails_fast_instead_of client: Final = _FakeRedisHoldingIncrements(FakeServer()) redis: Final = _RedisCacheOnClient(client, breaker) _process(monkeypatch, redis, cached_spend=6.0) + read: Final = asyncio.Event() + opened: Final = asyncio.Event() + + async def _read_before_the_seed_started() -> None: + with spend_counter_batch_scope(redis, frozenset((counter,))): + await _read_cold(counter) + read.set() + await opened.wait() + await ps.ensure_spend_counter_initialized(counter_key=counter, source_cache_key=_COLD_SEED_SOURCE) + + later: Final = asyncio.ensure_future(_read_before_the_seed_started()) + await read.wait() first: Final = asyncio.ensure_future(_cold_start(redis, counter)) await client.holding.wait() breaker.record_failure() - - later: Final = asyncio.ensure_future(_cold_start(redis, counter)) + opened.set() for _ in range(100): await asyncio.sleep(0) client.release.set() @@ -2453,64 +2472,6 @@ async def test_a_seed_that_lands_while_a_cold_read_is_on_its_way_back_is_not_add assert redis.values[counter] == 6.0 -class _HeldReadRedis(_SharedRedis): - """Answers a GET with the value its key had when it arrived, holding the reply while ``hold_reads`` is set.""" - - def __init__(self) -> None: - super().__init__() - self.hold_reads = False - self.read_arrived: Final = asyncio.Event() - self.release_read: Final = asyncio.Event() - self.reads_after_hold = 0 - - async def async_batch_get_cache(self, key_list: list[str], **_: object) -> dict[str, float | None]: - self.reads_after_hold += self.hold_reads - return await super().async_batch_get_cache(key_list) - - async def async_get_cache(self, key: str, **_: object) -> float | None: - self.reads_after_hold += self.hold_reads - value: Final = self.values.get(key) - if self.hold_reads: - self.read_arrived.set() - await self.release_read.wait() - return value - - -@pytest.mark.asyncio -async def test_a_seed_that_lands_while_a_request_re_reads_the_counter_is_not_added_again( - monkeypatch: pytest.MonkeyPatch, -) -> None: - counter: Final = "spend:end_user:seed-during-reread" - redis: Final = _HeldReadRedis() - _process(monkeypatch, redis, cached_spend=6.0) - read: Final = asyncio.Event() - seeded: Final = asyncio.Event() - - async def _stale_request() -> None: - with spend_counter_batch_scope(cast(RedisCache, redis), frozenset((counter,))): - await _read_cold(counter) - read.set() - await seeded.wait() - del redis.values[counter] - redis.hold_reads = True - await ps.ensure_spend_counter_initialized(counter_key=counter, source_cache_key=_COLD_SEED_SOURCE) - - async def _first_request() -> None: - await read.wait() - await _cold_start(redis, counter) - seeded.set() - - async def _request_seeding_during_the_re_read() -> None: - await redis.read_arrived.wait() - redis.hold_reads = False - await _cold_start(redis, counter) - redis.release_read.set() - - await asyncio.gather(_stale_request(), _first_request(), _request_seeding_during_the_re_read()) - - assert redis.values[counter] == 6.0 - - class _HeldGetRedis(_SharedRedis): """Answers the first GET with the value its key had when it arrived, holding the reply until released.""" @@ -2598,11 +2559,11 @@ async def test_a_cold_read_sent_before_an_add_started_does_not_join_it_when_the_ @pytest.mark.asyncio -async def test_a_re_read_redis_does_not_answer_stops_at_the_wait_budget_and_the_request_adds_its_cached_spend( +async def test_a_counter_lost_and_re_created_below_the_seed_gets_the_cached_spend_on_top( monkeypatch: pytest.MonkeyPatch, ) -> None: - counter: Final = "spend:end_user:stalled-reread" - redis: Final = _HeldReadRedis() + counter: Final = "spend:end_user:re-created-below" + redis: Final = _SharedRedis() _process(monkeypatch, redis, cached_spend=6.0) read: Final = asyncio.Event() seeded: Final = asyncio.Event() @@ -2612,8 +2573,36 @@ async def test_a_re_read_redis_does_not_answer_stops_at_the_wait_budget_and_the_ await _read_cold(counter) read.set() await seeded.wait() - del redis.values[counter] - redis.hold_reads = True + redis.values[counter] = 0.5 + await ps.ensure_spend_counter_initialized(counter_key=counter, source_cache_key=_COLD_SEED_SOURCE) + assert await read_batched_spend_counter(counter) == (6.5, True) + + async def _first_request() -> None: + await read.wait() + await _cold_start(redis, counter) + seeded.set() + + await asyncio.gather(_stale_request(), _first_request()) + + assert redis.values[counter] == 6.5 + + +@pytest.mark.asyncio +async def test_a_counter_holding_the_seed_is_left_alone_by_a_request_whose_source_cache_moved_past_it( + monkeypatch: pytest.MonkeyPatch, +) -> None: + counter: Final = "spend:end_user:source-moved" + redis: Final = _SharedRedis() + counters: Final = _process(monkeypatch, redis, cached_spend=6.0) + read: Final = asyncio.Event() + seeded: Final = asyncio.Event() + + async def _stale_request() -> None: + with spend_counter_batch_scope(cast(RedisCache, redis), frozenset((counter,))): + await _read_cold(counter) + read.set() + await seeded.wait() + ps.user_api_key_cache.in_memory_cache.set_cache(key=_COLD_SEED_SOURCE, value={"spend": 6.25}) await ps.ensure_spend_counter_initialized(counter_key=counter, source_cache_key=_COLD_SEED_SOURCE) async def _first_request() -> None: @@ -2621,10 +2610,153 @@ async def test_a_re_read_redis_does_not_answer_stops_at_the_wait_budget_and_the_ await _cold_start(redis, counter) seeded.set() - await asyncio.wait_for(asyncio.gather(_stale_request(), _first_request()), timeout=2.0) + await asyncio.gather(_stale_request(), _first_request()) assert redis.values[counter] == 6.0 - assert redis.reads_after_hold == 1 + assert counters.in_memory_cache.get_cache(key=counter) == 6.0 + + +@pytest.mark.asyncio +async def test_a_request_done_waiting_on_a_stalled_seed_adds_when_the_counter_holds_less_than_its_cached_spend( + monkeypatch: pytest.MonkeyPatch, +) -> None: + counter: Final = "spend:end_user:below-own-spend" + monkeypatch.setattr(spend_counter_reseed, "SPEND_COUNTER_COLD_SEED_JOIN_SECONDS", 0.01) + redis: Final = _HeldIncrementRedis(held_increments=1) + _process(monkeypatch, redis, cached_spend=6.0) + stalled: Final = asyncio.ensure_future(_cold_start(redis, counter)) + await redis.arrived[0].wait() + + with spend_counter_batch_scope(cast(RedisCache, redis), frozenset((counter,))): + await _read_cold(counter) + redis.values[counter] = 0.5 + await ps.ensure_spend_counter_initialized(counter_key=counter, source_cache_key=_COLD_SEED_SOURCE) + while_stalled: Final = redis.values[counter] + redis.releases[0].set() + await stalled + + assert while_stalled == 6.5 + + +@pytest.mark.asyncio +async def test_a_counter_without_redis_set_only_if_missing_keeps_the_value_it_holds( + monkeypatch: pytest.MonkeyPatch, +) -> None: + counter: Final = "spend:end_user:no-redis-set-if-missing" + monkeypatch.setattr(ps, "spend_counter_cache", DualCache(in_memory_cache=InMemoryCache())) + + first: Final = await ps.increment_spend_counter_cache(counter, 6.0, only_if_missing=True) + second: Final = await ps.increment_spend_counter_cache(counter, 4.0, only_if_missing=True) + added: Final = await ps.increment_spend_counter_cache(counter, 1.0) + + assert (first, second, added) == (6.0, 6.0, 7.0) + + +@pytest.mark.asyncio +async def test_a_counter_set_by_a_request_done_waiting_counts_as_the_seed_for_a_later_request( + monkeypatch: pytest.MonkeyPatch, +) -> None: + counter: Final = "spend:end_user:set-counts-as-seed" + monkeypatch.setattr(spend_counter_reseed, "SPEND_COUNTER_COLD_SEED_JOIN_SECONDS", 0.01) + redis: Final = _HeldIncrementRedis(held_increments=1) + _process(monkeypatch, redis, cached_spend=6.0) + read: Final = asyncio.Event() + set_by_a_waiter: Final = asyncio.Event() + + async def _stale_request() -> None: + with spend_counter_batch_scope(cast(RedisCache, redis), frozenset((counter,))): + await _read_cold(counter) + read.set() + await set_by_a_waiter.wait() + redis.values[counter] = 0.5 + await ps.ensure_spend_counter_initialized(counter_key=counter, source_cache_key=_COLD_SEED_SOURCE) + + stale: Final = asyncio.ensure_future(_stale_request()) + await read.wait() + stalled: Final = asyncio.ensure_future(_cold_start(redis, counter)) + await redis.arrived[0].wait() + await _cold_start(redis, counter) + set_by_a_waiter.set() + await stale + after_the_stale_request: Final = redis.values[counter] + redis.releases[0].set() + await stalled + + assert after_the_stale_request == 6.5 + + +class _CommandCountingRedis(_SharedRedis): + def __init__(self) -> None: + super().__init__() + self.commands: Final[list[str]] = [] # mutable-ok: the commands the test asserts on + + async def async_batch_get_cache(self, key_list: list[str], **_: object) -> dict[str, float | None]: + self.commands.append("mget") + return await super().async_batch_get_cache(key_list) + + async def async_get_cache(self, key: str, **_: object) -> float | None: + self.commands.append("get") + return await super().async_get_cache(key) + + async def async_increment(self, key: str, value: float, **kwargs: bool) -> float: + self.commands.append("set_if_missing" if kwargs.get("only_if_missing") else "add") + return await super().async_increment(key, value, **kwargs) + + +@pytest.mark.asyncio +async def test_a_request_that_read_before_a_seed_landed_sends_one_write_and_keeps_its_value_for_later_reads( + monkeypatch: pytest.MonkeyPatch, +) -> None: + counter: Final = "spend:end_user:one-write" + redis: Final = _CommandCountingRedis() + _process(monkeypatch, redis, cached_spend=6.0) + read: Final = asyncio.Event() + seeded: Final = asyncio.Event() + + async def _stale_request() -> None: + with spend_counter_batch_scope(cast(RedisCache, redis), frozenset((counter,))): + await _read_cold(counter) + read.set() + await seeded.wait() + redis.commands.clear() + await ps.ensure_spend_counter_initialized(counter_key=counter, source_cache_key=_COLD_SEED_SOURCE) + assert await read_batched_spend_counter(counter) == (6.0, True) + assert await ps.get_current_spend(counter_key=counter, fallback_spend=6.0) == 6.0 + + async def _first_request() -> None: + await read.wait() + await _cold_start(redis, counter) + seeded.set() + + await asyncio.gather(_stale_request(), _first_request()) + + assert redis.commands == ["set_if_missing"] + + +@pytest.mark.asyncio +async def test_a_request_waits_on_other_requests_adds_for_the_limit_in_total_across_its_counters( + monkeypatch: pytest.MonkeyPatch, +) -> None: + limit: Final = 0.3 + monkeypatch.setattr(spend_counter_reseed, "SPEND_COUNTER_COLD_SEED_JOIN_SECONDS", limit) + keys: Final = tuple(f"spend:end_user:shared-wait-{index}" for index in range(4)) + redis: Final = _HeldIncrementRedis(held_increments=len(keys)) + _process(monkeypatch, redis, cached_spend=6.0) + stalled: Final = [asyncio.ensure_future(_cold_start(redis, key)) for key in keys] + for arrived in redis.arrived: + await arrived.wait() + + loop: Final = asyncio.get_running_loop() + started: Final = loop.time() + with spend_counter_batch_scope(cast(RedisCache, redis), frozenset(keys)): + for key in keys: + await ps.ensure_spend_counter_initialized(counter_key=key, source_cache_key=_COLD_SEED_SOURCE) + waited: Final = loop.time() - started + for release in redis.releases: + release.set() + await asyncio.gather(*stalled) + + assert limit <= waited < 2 * limit class _LostAndRecreatedAfterFailedMGetRedis(_SharedRedis): @@ -2657,9 +2789,9 @@ class _LostAndRecreatedAfterFailedMGetRedis(_SharedRedis): self.cold_gets += 1 return await super().async_get_cache(key) - async def async_increment(self, key: str, value: float, **_: object) -> float: + async def async_increment(self, key: str, value: float, **kwargs: bool) -> float: self._other_process_increments_after_the_cold_gets() - return await super().async_increment(key, value) + return await super().async_increment(key, value, **kwargs) @pytest.mark.asyncio @@ -2686,8 +2818,8 @@ class _AppliedHeldThenResetRedis(_HeldReplyRedis): super().__init__() self.deleted = False - async def async_increment(self, key: str, value: float, **_: object) -> float: - await super().async_increment(key, value) + async def async_increment(self, key: str, value: float, **kwargs: bool) -> float: + await super().async_increment(key, value, **kwargs) raise ConnectionError("connection reset") async def async_delete_cache(self, key: str) -> None: @@ -2721,6 +2853,7 @@ async def test_requests_waiting_on_a_cold_seed_whose_request_was_cancelled_add_t monkeypatch: pytest.MonkeyPatch, ) -> None: counter: Final = "spend:end_user:cancelled-seed-with-waiters" + monkeypatch.setattr(spend_counter_reseed, "SPEND_COUNTER_COLD_SEED_JOIN_SECONDS", 60.0) redis: Final = _StalledFirstIncrementRedis() _process(monkeypatch, redis, cached_spend=6.0) read: Final = asyncio.Event() @@ -2742,7 +2875,7 @@ async def test_requests_waiting_on_a_cold_seed_whose_request_was_cancelled_add_t for _ in range(20): await asyncio.sleep(0) seed.cancel() - await asyncio.gather(joined_before_start, joined_after_start) + await asyncio.wait_for(asyncio.gather(joined_before_start, joined_after_start), timeout=5) assert seed.cancelled() assert redis.values[counter] == 6.0 @@ -2766,13 +2899,13 @@ class _HeldIncrementRedis(_SharedRedis): self.reads += 1 return await super().async_get_cache(key) - async def async_increment(self, key: str, value: float, **_: object) -> float: + async def async_increment(self, key: str, value: float, **kwargs: bool) -> float: index: Final = self.increments self.increments += 1 if index < len(self.arrived): self.arrived[index].set() await self.releases[index].wait() - return await super().async_increment(key, value) + return await super().async_increment(key, value, **kwargs) @pytest.mark.asyncio @@ -2830,23 +2963,11 @@ async def test_a_request_that_read_the_counter_while_a_seed_was_in_flight_does_n assert redis.values[counter] == 6.0 -class _SlowGetRedis(_HeldIncrementRedis): - """Also takes ``SECONDS`` to answer a GET.""" - - SECONDS: ClassVar[float] = 0.2 - - async def async_get_cache(self, key: str, **_: object) -> float | None: - await asyncio.sleep(self.SECONDS) - return await super().async_get_cache(key) - - @pytest.mark.asyncio -async def test_a_seed_that_lands_late_in_the_wait_leaves_time_to_re_read_the_counter_instead_of_adding_again( - monkeypatch: pytest.MonkeyPatch, -) -> None: +async def test_a_seed_that_lands_late_in_the_wait_is_not_added_again(monkeypatch: pytest.MonkeyPatch) -> None: counter: Final = "spend:end_user:late-landing-seed" monkeypatch.setattr(spend_counter_reseed, "SPEND_COUNTER_COLD_SEED_JOIN_SECONDS", 0.5) - redis: Final = _SlowGetRedis(held_increments=1) + redis: Final = _HeldIncrementRedis(held_increments=1) _process(monkeypatch, redis, cached_spend=6.0) first: Final = asyncio.ensure_future(_cold_start(redis, counter)) @@ -2859,6 +2980,35 @@ async def test_a_seed_that_lands_late_in_the_wait_leaves_time_to_re_read_the_cou assert redis.values[counter] == 6.0 +@pytest.mark.asyncio +async def test_a_request_joining_an_add_that_started_after_its_read_came_back_sends_no_write( + monkeypatch: pytest.MonkeyPatch, +) -> None: + counter: Final = "spend:end_user:joined-no-write" + monkeypatch.setattr(spend_counter_reseed, "SPEND_COUNTER_COLD_SEED_JOIN_SECONDS", 60.0) + redis: Final = _HeldIncrementRedis(held_increments=1) + _process(monkeypatch, redis, cached_spend=6.0) + read: Final = asyncio.Event() + + async def _joining_request() -> None: + with spend_counter_batch_scope(cast(RedisCache, redis), frozenset((counter,))): + await _read_cold(counter) + read.set() + await redis.arrived[0].wait() + await ps.ensure_spend_counter_initialized(counter_key=counter, source_cache_key=_COLD_SEED_SOURCE) + + joining: Final = asyncio.ensure_future(_joining_request()) + await read.wait() + first: Final = asyncio.ensure_future(_cold_start(redis, counter)) + await redis.arrived[0].wait() + for _ in range(100): + await asyncio.sleep(0) + redis.releases[0].set() + await asyncio.gather(joining, first) + + assert (redis.values[counter], redis.increments) == (6.0, 1) + + class _HeldSecondGetRedis(_HeldIncrementRedis): """Also answers the second GET with the value its key had when it arrived, holding the reply until released.""" @@ -2917,7 +3067,63 @@ async def test_a_stalled_seed_finishing_does_not_hide_the_newer_seed_still_in_fl late: Final = asyncio.ensure_future(_cold_start(redis, counter)) for _ in range(10): await asyncio.sleep(0) + commands_while_the_newer_seed_was_in_flight: Final = redis.increments redis.releases[1].set() await asyncio.gather(newer, late) + assert commands_while_the_newer_seed_was_in_flight == 2 assert redis.values[counter] == 6.0 + + +class _TargetRecordingRedis(_SharedRedis): + def __init__(self) -> None: + super().__init__() + self.targets: Final[list[str | None]] = [] # mutable-ok: the targets the test asserts on + + async def async_get_cache(self, key: str, **_: object) -> float | None: + self.targets.append(current_service_target()) + return 1.0 + + +@pytest.mark.asyncio +async def test_the_cold_check_read_reports_under_spend_counters_inside_another_phases_target( + monkeypatch: pytest.MonkeyPatch, +) -> None: + redis: Final = _TargetRecordingRedis() + _process(monkeypatch, redis, cached_spend=6.0) + + with service_target(AUTH_OBJECTS_TARGET): + await ps.ensure_spend_counter_initialized(counter_key="spend:key:target", source_cache_key=_COLD_SEED_SOURCE) + + assert redis.targets == [SPEND_COUNTERS_TARGET] + + +@pytest.mark.asyncio +async def test_a_stale_read_on_a_real_client_seeds_a_missing_counter_with_its_ttl_and_keeps_an_existing_one( + monkeypatch: pytest.MonkeyPatch, +) -> None: + counter: Final = "spend:end_user:real-client" + client: Final = FakeAsyncRedis(server=FakeServer()) + redis: Final = _RedisCacheOnClient(client) + redis.default_ttl = 120 + _process(monkeypatch, redis, cached_spend=6.0) + read: Final = asyncio.Event() + seeded: Final = asyncio.Event() + + async def _stale_request() -> None: + with spend_counter_batch_scope(redis, frozenset((counter,))): + await _read_cold(counter) + read.set() + await seeded.wait() + await client.delete(counter) + await ps.ensure_spend_counter_initialized(counter_key=counter, source_cache_key=_COLD_SEED_SOURCE) + + async def _first_request() -> None: + await read.wait() + await _cold_start(redis, counter) + seeded.set() + + await asyncio.gather(_stale_request(), _first_request()) + + assert float(await client.get(counter) or 0) == 6.0 + assert 0 < await client.ttl(counter) <= 120