fix(proxy): set a missing cold counter with SET NX instead of a timed re-read

A request whose cold read may predate this process's add now sends SET NX, EXPIRE and GET in one pipeline, the same shape as main's INCRBYFLOAT and EXPIRE, so it sends one write and no read, cancels nothing, and needs no second time limit

It adds on top only when the counter holds less than what the landed add should have left, which keeps main's sum when a counter was lost and re-created below that

The wait on other requests' adds now shares one limit across a request's counters, and the cold check read goes back to main's decorated helper so its span and service target stay spend_counters
This commit is contained in:
yucheng 2026-11-02 15:51:37 -08:00
parent 2e83346cb2
commit 0ad56125d2
6 changed files with 475 additions and 168 deletions

View file

@ -1473,15 +1473,29 @@ class RedisCache(BaseCache):
await _redis_client.expire(key, ttl)
return float(result)
async def _set_if_missing_with_ttl(self, key: str, value: float, ttl: int | None) -> float:
"""SET NX, its TTL command and a GET in one round trip; the value the key holds afterwards."""
async with self._async_commands().pipeline(transaction=False) as pipe:
pipe.set(name=key, value=value, nx=True)
if ttl is not None:
pipe.expire(key, ttl)
pipe.get(key)
replies: Final[Sequence[object]] = await pipe.execute() # pyright: ignore[reportUnknownVariableType, reportUnknownMemberType] # redis generics
current: Final = _LUA_OPTIONAL_TOTAL.validate_python(replies[-1])
return value if current is None else current
@_redis_circuit_breaker_guard
async def async_increment(
self,
key,
key: str,
value: float,
ttl: int | None = None,
parent_otel_span: Span | None = None,
refresh_ttl: bool = False,
only_if_missing: bool = False,
) -> float:
"""Add ``value`` to the key and return its new value. With ``only_if_missing``, a key that exists keeps its
value, which is returned, and only the TTL is refreshed."""
from redis.asyncio import Redis
_redis_client: Final[Redis] = self.init_async_client()
@ -1489,8 +1503,12 @@ class RedisCache(BaseCache):
_used_ttl: Final = self.get_ttl(ttl=ttl)
key = self.check_and_fix_namespace(key=key)
try:
result: Final = await self._incrbyfloat_with_ttl(
_redis_client, key=key, value=value, ttl=_used_ttl, refresh_ttl=refresh_ttl
result: Final = await (
self._set_if_missing_with_ttl(key=key, value=value, ttl=_used_ttl)
if only_if_missing
else self._incrbyfloat_with_ttl(
_redis_client, key=key, value=value, ttl=_used_ttl, refresh_ttl=refresh_ttl
)
)
## LOGGING ##

View file

@ -16,6 +16,7 @@ import asyncio
import functools
from collections import OrderedDict
from collections.abc import Awaitable, Callable, Mapping
from contextvars import ContextVar
from dataclasses import dataclass
from datetime import datetime, timezone
from types import MappingProxyType
@ -30,7 +31,6 @@ from litellm.proxy.db.db_lookup_gate import bounded_db_lookup, db_lookup_gate
from litellm.proxy.spend_tracking.spend_counter_batch import (
SPEND_COUNTERS_TARGET,
SpendCounterRead,
forget_spend_counter,
read_batched_spend_counter,
record_spend_counter_value,
spend_counter_tick,
@ -75,11 +75,12 @@ _WINDOW_SPEND_LOG_FIELDS: Final[Mapping[str, str]] = MappingProxyType(
@dataclass(slots=True)
class _CachedSpendSeed:
"""This process adding its cached spend to one cold counter: the add in flight, the tick it started at, and the
tick the last one landed at."""
"""This process adding its cached spend to one cold counter: the add in flight, the tick it started at, the tick
the last one landed at, and the least the counter held when that one landed."""
started_at: int = 0
landed_at: int = 0
landed_spend: float = 0.0
in_flight: asyncio.Future[float | None] | None = None
def on_this_loop(self) -> bool:
@ -107,6 +108,18 @@ def _cached_spend_seed(spend_counter_cache: "DualCache", counter_key: str) -> _C
return _CachedSpendSeed()
_cold_seed_waited: Final[ContextVar[float]] = ContextVar("cold_spend_seed_waited", default=0.0)
async def _wait_for_cold_seed(in_flight: asyncio.Future[float | None]) -> None:
"""Wait on ``in_flight`` for what this context has left of ``SPEND_COUNTER_COLD_SEED_JOIN_SECONDS``."""
loop: Final = asyncio.get_running_loop()
waited: Final = _cold_seed_waited.get()
started: Final = loop.time()
await asyncio.wait((in_flight,), timeout=max(0.0, SPEND_COUNTER_COLD_SEED_JOIN_SECONDS - waited))
_cold_seed_waited.set(waited + loop.time() - started)
def _redis_breaker_closed(spend_counter_cache: "DualCache") -> bool:
redis_cache: Final = spend_counter_cache.redis_cache
return redis_cache is None or redis_cache.circuit_breaker_closed
@ -161,56 +174,66 @@ class SpendCounterReseed:
spend_counter_cache: "DualCache",
counter_key: str,
read: SpendCounterRead,
spend: float,
add: Callable[[], Awaitable[float | None]],
is_warm: Callable[[], Awaitable[bool]],
set_if_missing: Callable[[], Awaitable[float | None]],
) -> None:
"""Run ``add``, this process adding its cached spend to a cold counter, once for all of this process's
requests that read the counter cold before that add landed. A request joins an add that started after its read
came back and gets its result, and starts or joins another add when that one fails or is cancelled. When an add
landed after its read went out, or one that started before its read came back finishes, the request re-reads
the counter and adds again only if the counter is gone. A request waits on other requests' adds for at most
``SPEND_COUNTER_COLD_SEED_JOIN_SECONDS`` in all, has that long again to finish a re-read, and then adds its own.
A request on another event loop than the add in flight, one that finds the Redis circuit breaker not closed, and
each other process, still add their own cached spend."""
"""Run ``add``, this process adding ``spend`` to a cold counter, once for all of this process's requests that
read the counter cold before that add landed. A request joins an add that started after its read came back and
gets its result, and starts or joins another add when that one fails or is cancelled. A request whose read went
out before an add landed, and one still waiting on an add after ``SPEND_COUNTER_COLD_SEED_JOIN_SECONDS`` across
its counters, runs ``set_if_missing`` in place of ``add``, then runs ``add`` too if the counter holds less than
the spend it should already hold. A request on another event loop than the add in flight, one that finds the
Redis circuit breaker not closed, and each other process, still run ``add``."""
seed: Final = _cached_spend_seed(spend_counter_cache, counter_key)
loop: Final = asyncio.get_running_loop()
join_by: Final = loop.time() + SPEND_COUNTER_COLD_SEED_JOIN_SECONDS
re_read_by: Final = join_by + SPEND_COUNTER_COLD_SEED_JOIN_SECONDS
async def add_unless_covered(cold: SpendCounterRead) -> float | None:
async def add_spend() -> float | None:
added: Final = await add()
seed.landed_spend = spend
return added
async def add_if_short() -> float | None:
least: Final = min(spend, seed.landed_spend) if seed.landed_at >= read.sent_at else spend
current: Final = await set_if_missing()
if current is not None and current < least:
return await add_spend()
seed.landed_spend = least
return current
async def add_unless_covered() -> float | None:
in_flight: Final = seed.in_flight
if not seed.on_this_loop() or not _redis_breaker_closed(spend_counter_cache):
return await add()
if in_flight is None:
if seed.landed_at < cold.sent_at:
return await seed.run(add)
forget_spend_counter(counter_key)
sent_at: Final = spend_counter_tick()
try:
warm: Final = await asyncio.wait_for(is_warm(), timeout=max(0.0, re_read_by - loop.time()))
except asyncio.TimeoutError:
return await seed.run(add)
if warm:
return None
return await add_unless_covered(SpendCounterRead(sent_at=sent_at, returned_at=spend_counter_tick()))
covers: Final = seed.started_at > cold.returned_at
await asyncio.wait((in_flight,), timeout=max(0.0, join_by - loop.time()))
return await seed.run(add_spend if seed.landed_at < read.sent_at else add_if_short)
covers: Final = seed.started_at > read.returned_at
await _wait_for_cold_seed(in_flight)
if not in_flight.done():
return await seed.run(add)
return await seed.run(add_if_short)
if covers and not in_flight.cancelled():
return in_flight.result()
return await add_unless_covered(cold)
return await add_unless_covered()
added: Final = await add_unless_covered(read)
added: Final = await add_unless_covered()
if added is not None:
record_spend_counter_value(counter_key, added)
@staticmethod
@with_service_target(SPEND_COUNTERS_TARGET)
async def increment_in_memory(spend_counter_cache: "DualCache", counter_key: str, increment: float) -> float | None:
"""Apply local deltas after an in-flight reseed establishes the spend balance."""
async def increment_in_memory(
spend_counter_cache: "DualCache", counter_key: str, increment: float, only_if_missing: bool = False
) -> float | None:
"""Apply local deltas after an in-flight reseed establishes the spend balance. With ``only_if_missing``, a
counter that exists keeps its value, which is returned."""
lock: Final = await SpendCounterReseed._get_lock(counter_key)
async with lock:
current: Final[float | None] = ( # pyright: ignore[reportUnknownVariableType] # untyped in-memory cache
spend_counter_cache.in_memory_cache.get_cache(key=counter_key) # pyright: ignore[reportUnknownMemberType] # untyped in-memory cache
if only_if_missing
else None
)
if current is not None:
return float(current)
return await spend_counter_cache.async_increment_cache(
key=counter_key, value=increment, local_only=True, refresh_ttl=True
)

View file

@ -930,7 +930,6 @@ from litellm.proxy.spend_tracking.spend_capture_rate import (
from litellm.proxy.spend_tracking.spend_counter_batch import (
SPEND_COUNTERS_TARGET,
PendingSpendIncrement,
SpendCounterRead,
active_spend_counter_batch,
forget_spend_counter,
post_call_counter_keys,
@ -4087,8 +4086,9 @@ async def ensure_spend_counter_initialized(
counter_key: str,
source_cache_key: str | list[str],
) -> None:
read: Final = await _cold_spend_counter_read(counter_key)
if read is not None:
is_warm: Final = await _is_spend_counter_cache_warm(counter_key=counter_key)
if is_warm is False:
read: Final = spend_counter_read(counter_key)
# Shares the per-counter lock with get_current_spend.
db_spend: Final = await SpendCounterReseed.coalesced(
prisma_client=prisma_client,
@ -4104,8 +4104,11 @@ async def ensure_spend_counter_initialized(
spend_counter_cache=spend_counter_cache,
counter_key=counter_key,
read=read,
spend=base_spend,
add=lambda: increment_spend_counter_cache(counter_key=counter_key, increment=base_spend),
is_warm=lambda: _is_spend_counter_warm_outside_batch(counter_key),
set_if_missing=lambda: increment_spend_counter_cache(
counter_key=counter_key, increment=base_spend, only_if_missing=True
),
)
@ -4160,22 +4163,14 @@ _ensure_window_spend_counter_initialized: Final = ensure_window_spend_counter_in
@with_service_target(SPEND_COUNTERS_TARGET)
async def _is_spend_counter_cache_warm(counter_key: str) -> bool:
return await _cold_spend_counter_read(counter_key) is None
async def _cold_spend_counter_read(counter_key: str) -> SpendCounterRead | None:
"""When the read that found the counter missing went out and came back, None when the counter has a value. A read
outside the batch counts as now, since ``SpendCounterReseed.coalesced`` reads the counter again after it."""
batched: Final = await read_batched_spend_counter(counter_key)
if batched is not None and batched[0] is not None:
spend_counter_cache.in_memory_cache.set_cache(key=counter_key, value=batched[0])
return None
if batched is None and await _is_spend_counter_warm_outside_batch(counter_key):
return None
return spend_counter_read(counter_key)
if batched is not None:
batched_value, _ = batched
if batched_value is None:
return False
spend_counter_cache.in_memory_cache.set_cache(key=counter_key, value=batched_value)
return True
async def _is_spend_counter_warm_outside_batch(counter_key: str) -> bool:
if spend_counter_cache.redis_cache is not None:
try:
current_value: Final[object] = await spend_counter_cache.redis_cache.async_get_cache(
@ -4217,13 +4212,24 @@ async def refresh_spend_counter_ttl(counter_key: str) -> bool:
@with_service_target(SPEND_COUNTERS_TARGET)
async def increment_spend_counter_cache(counter_key: str, increment: float) -> float | None:
async def increment_spend_counter_cache(
counter_key: str, increment: float, only_if_missing: bool = False
) -> float | None:
if spend_counter_cache.redis_cache is not None:
try:
current_value: Final = await spend_counter_cache.redis_cache.async_increment(
key=counter_key,
value=increment,
refresh_ttl=True,
current_value: Final = await (
spend_counter_cache.redis_cache.async_increment(
key=counter_key,
value=increment,
refresh_ttl=True,
only_if_missing=True,
)
if only_if_missing
else spend_counter_cache.redis_cache.async_increment(
key=counter_key,
value=increment,
refresh_ttl=True,
)
)
except Exception:
await invalidate_spend_counter(counter_key=counter_key)
@ -4236,7 +4242,10 @@ async def increment_spend_counter_cache(counter_key: str, increment: float) -> f
return current_value
return await SpendCounterReseed.increment_in_memory(
spend_counter_cache=spend_counter_cache, counter_key=counter_key, increment=increment
spend_counter_cache=spend_counter_cache,
counter_key=counter_key,
increment=increment,
only_if_missing=only_if_missing,
)

View file

@ -188,7 +188,8 @@ def active_spend_counter_batch() -> SpendCounterBatch | None:
def spend_counter_read(counter_key: str) -> SpendCounterRead:
"""When this context's read of the counter went out and came back: the open batch's MGET for it, else now."""
"""When this context's read of the counter went out and came back: the open batch's MGET for it, else now, since
``SpendCounterReseed.coalesced`` reads the counter again after a read outside the batch."""
batch: Final = _active_batch.get()
batched: Final = batch.counter_read(counter_key) if batch is not None else None
if batched is not None:

View file

@ -8,6 +8,7 @@ from typing import Final, cast
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from fakeredis import FakeAsyncRedis, FakeServer
import litellm
from litellm._service_logger import ServiceLogging
@ -1474,6 +1475,55 @@ async def test_async_increment_pipelines_the_ttl_command(
assert client.round_trips == expected_round_trips
@pytest.mark.asyncio
@pytest.mark.parametrize(
("existing", "expected"),
[
pytest.param(None, 1.5, id="missing key: set to the value"),
pytest.param(0.25, 0.25, id="key below the value: kept"),
pytest.param(4.0, 4.0, id="key above the value: kept"),
],
)
async def test_async_increment_only_if_missing_sets_a_missing_key_and_refreshes_the_ttl(
monkeypatch, redis_no_ping, existing, expected
):
monkeypatch.setenv("REDIS_HOST", "https://my-test-host")
redis_cache = RedisCache(namespace="ns")
client = FakeAsyncRedis(server=FakeServer())
if existing is not None:
await client.set("ns:spend:key:k", existing, ex=5)
with patch.object(redis_cache, "init_async_client", return_value=client):
current = await redis_cache.async_increment(
key="spend:key:k", value=1.5, ttl=60, refresh_ttl=True, only_if_missing=True
)
after_add = await redis_cache.async_increment(key="spend:key:k", value=2.0, ttl=60, refresh_ttl=True)
assert current == expected
assert after_add == expected + 2.0
assert await client.ttl("ns:spend:key:k") == 60
@pytest.mark.asyncio
async def test_async_increment_only_if_missing_reports_a_redis_failure_like_an_increment(monkeypatch, redis_no_ping):
monkeypatch.setenv("REDIS_HOST", "https://my-test-host")
redis_cache = RedisCache(namespace="ns")
client = MagicMock()
client.pipeline.side_effect = ConnectionError("connection reset")
failure_hook = AsyncMock()
with (
patch.object(redis_cache, "init_async_client", return_value=client),
patch.object(redis_cache.service_logger_obj, "async_service_failure_hook", failure_hook),
pytest.raises(ConnectionError),
):
await redis_cache.async_increment(key="spend:key:k", value=1.5, ttl=60, refresh_ttl=True, only_if_missing=True)
await asyncio.sleep(0)
assert failure_hook.await_args.kwargs["call_type"] == "async_increment"
assert redis_cache._circuit_breaker._hard_failure_count == 1
class _SetRecordingPipeline:
def __init__(self) -> None:
self.sets: list[tuple[str, str, timedelta | None]] = []

View file

@ -25,7 +25,7 @@ import asyncio
import threading
from collections.abc import Awaitable, Callable
from datetime import datetime
from typing import ClassVar, Final, Literal, cast
from typing import Final, Literal, cast
from unittest.mock import AsyncMock, MagicMock, call
import pytest
@ -37,13 +37,13 @@ from opentelemetry.trace import StatusCode
import litellm
import litellm.proxy.proxy_server as ps
from litellm._internal_context import current_service_target
from litellm._internal_context import current_service_target, service_target
from litellm._service_logger import ServiceLogging
from litellm.caching.dual_cache import DualCache
from litellm.caching.in_memory_cache import InMemoryCache
from litellm.caching.redis_batch import request_redis_batch_scope
from litellm.caching.redis_cache import RedisCache, RedisCircuitBreaker, RedisCircuitBreakerOpenError
from litellm.constants import GLOBAL_PROXY_SPEND_CACHE_KEY
from litellm.constants import AUTH_OBJECTS_TARGET, GLOBAL_PROXY_SPEND_CACHE_KEY
from litellm.integrations.opentelemetry import OpenTelemetry, OpenTelemetryConfig
from litellm.models.end_user import LiteLLM_EndUserTable
from litellm.models.tag import LiteLLM_TagTable
@ -55,6 +55,7 @@ from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache, end_u
from litellm.proxy.db import spend_counter_reseed
from litellm.proxy.management_endpoints.team_endpoints import refresh_cached_team
from litellm.proxy.spend_tracking.spend_counter_batch import (
SPEND_COUNTERS_TARGET,
active_spend_counter_batch,
read_batched_spend_counter,
spend_counter_batch_scope,
@ -2044,12 +2045,16 @@ class _SharedRedis:
await asyncio.sleep(0)
return self.values.get(key)
async def async_increment(self, key: str, value: float, **_: object) -> float:
def _apply(self, key: str, value: float, only_if_missing: bool = False, **_: object) -> float:
if not only_if_missing or key not in self.values:
self.values[key] = self.values.get(key, 0.0) + value
return self.values[key]
async def async_increment(self, key: str, value: float, **kwargs: bool) -> float:
await asyncio.sleep(0)
if self.failing:
raise ConnectionError("redis down")
self.values[key] = self.values.get(key, 0.0) + value
return self.values[key]
return self._apply(key, value, **kwargs)
async def async_delete_cache(self, key: str) -> None:
await asyncio.sleep(0)
@ -2091,7 +2096,9 @@ async def test_overlapping_cold_starts_in_one_process_add_its_cached_spend_once(
await asyncio.gather(*(_cold_start(redis, counter) for _ in range(5)))
seeded: Final = redis.values.get(counter) if redis is not None else counters.in_memory_cache.get_cache(key=counter)
seeded: Final[float | None] = (
redis.values.get(counter) if redis is not None else counters.in_memory_cache.get_cache(key=counter)
)
assert seeded == 6.0
@ -2205,9 +2212,8 @@ class _HeldReplyRedis(_SharedRedis):
self.applied: Final = asyncio.Event()
self.release: Final = asyncio.Event()
async def async_increment(self, key: str, value: float, **_: object) -> float:
self.values[key] = self.values.get(key, 0.0) + value
total: Final = self.values[key]
async def async_increment(self, key: str, value: float, **kwargs: bool) -> float:
total: Final = self._apply(key, value, **kwargs)
self.applied.set()
await self.release.wait()
return total
@ -2240,12 +2246,12 @@ def test_a_cold_start_on_another_event_loop_adds_on_its_own(monkeypatch: pytest.
release: Final = threading.Event()
class _ThreadHeldRedis(_SharedRedis):
async def async_increment(self, key: str, value: float, **_: object) -> float:
async def async_increment(self, key: str, value: float, **kwargs: bool) -> float:
if threading.current_thread() is not threading.main_thread():
held.set()
while not release.is_set():
await asyncio.sleep(0.001)
return await super().async_increment(key, value)
return await super().async_increment(key, value, **kwargs)
redis: Final = _ThreadHeldRedis()
_process(monkeypatch, redis, cached_spend=6.0)
@ -2326,12 +2332,12 @@ class _DroppedFirstIncrementRedis(_SharedRedis):
super().__init__()
self.dropped = False
async def async_increment(self, key: str, value: float, **_: object) -> float:
async def async_increment(self, key: str, value: float, **kwargs: bool) -> float:
if not self.dropped:
self.dropped = True
await asyncio.sleep(0)
raise ConnectionError("connection reset by peer")
return await super().async_increment(key, value)
return await super().async_increment(key, value, **kwargs)
@pytest.mark.asyncio
@ -2357,16 +2363,18 @@ class _StalledFirstIncrementRedis(_SharedRedis):
self.stalled: Final = asyncio.Event()
self.release: Final = asyncio.Event()
async def async_increment(self, key: str, value: float, **_: object) -> float:
async def async_increment(self, key: str, value: float, **kwargs: bool) -> float:
if not self.stalled.is_set():
self.stalled.set()
await self.release.wait()
raise TimeoutError("redis did not answer")
return await super().async_increment(key, value)
return await super().async_increment(key, value, **kwargs)
@pytest.mark.asyncio
async def test_cold_starts_stop_waiting_on_a_stalled_seed_and_add_their_own(monkeypatch: pytest.MonkeyPatch) -> None:
async def test_cold_starts_stop_waiting_on_a_stalled_seed_and_set_the_missing_counter_once(
monkeypatch: pytest.MonkeyPatch,
) -> None:
counter: Final = "spend:end_user:stalled-seed"
redis: Final = _StalledFirstIncrementRedis()
_process(monkeypatch, redis, cached_spend=6.0)
@ -2381,7 +2389,7 @@ async def test_cold_starts_stop_waiting_on_a_stalled_seed_and_add_their_own(monk
with pytest.raises(TimeoutError):
await stalled
assert seeded_while_stalled == 18.0
assert seeded_while_stalled == 6.0
class _FakeRedisHoldingIncrements(FakeAsyncRedis):
@ -2405,11 +2413,22 @@ async def test_a_cold_start_after_the_redis_breaker_opened_fails_fast_instead_of
client: Final = _FakeRedisHoldingIncrements(FakeServer())
redis: Final = _RedisCacheOnClient(client, breaker)
_process(monkeypatch, redis, cached_spend=6.0)
read: Final = asyncio.Event()
opened: Final = asyncio.Event()
async def _read_before_the_seed_started() -> None:
with spend_counter_batch_scope(redis, frozenset((counter,))):
await _read_cold(counter)
read.set()
await opened.wait()
await ps.ensure_spend_counter_initialized(counter_key=counter, source_cache_key=_COLD_SEED_SOURCE)
later: Final = asyncio.ensure_future(_read_before_the_seed_started())
await read.wait()
first: Final = asyncio.ensure_future(_cold_start(redis, counter))
await client.holding.wait()
breaker.record_failure()
later: Final = asyncio.ensure_future(_cold_start(redis, counter))
opened.set()
for _ in range(100):
await asyncio.sleep(0)
client.release.set()
@ -2453,64 +2472,6 @@ async def test_a_seed_that_lands_while_a_cold_read_is_on_its_way_back_is_not_add
assert redis.values[counter] == 6.0
class _HeldReadRedis(_SharedRedis):
"""Answers a GET with the value its key had when it arrived, holding the reply while ``hold_reads`` is set."""
def __init__(self) -> None:
super().__init__()
self.hold_reads = False
self.read_arrived: Final = asyncio.Event()
self.release_read: Final = asyncio.Event()
self.reads_after_hold = 0
async def async_batch_get_cache(self, key_list: list[str], **_: object) -> dict[str, float | None]:
self.reads_after_hold += self.hold_reads
return await super().async_batch_get_cache(key_list)
async def async_get_cache(self, key: str, **_: object) -> float | None:
self.reads_after_hold += self.hold_reads
value: Final = self.values.get(key)
if self.hold_reads:
self.read_arrived.set()
await self.release_read.wait()
return value
@pytest.mark.asyncio
async def test_a_seed_that_lands_while_a_request_re_reads_the_counter_is_not_added_again(
monkeypatch: pytest.MonkeyPatch,
) -> None:
counter: Final = "spend:end_user:seed-during-reread"
redis: Final = _HeldReadRedis()
_process(monkeypatch, redis, cached_spend=6.0)
read: Final = asyncio.Event()
seeded: Final = asyncio.Event()
async def _stale_request() -> None:
with spend_counter_batch_scope(cast(RedisCache, redis), frozenset((counter,))):
await _read_cold(counter)
read.set()
await seeded.wait()
del redis.values[counter]
redis.hold_reads = True
await ps.ensure_spend_counter_initialized(counter_key=counter, source_cache_key=_COLD_SEED_SOURCE)
async def _first_request() -> None:
await read.wait()
await _cold_start(redis, counter)
seeded.set()
async def _request_seeding_during_the_re_read() -> None:
await redis.read_arrived.wait()
redis.hold_reads = False
await _cold_start(redis, counter)
redis.release_read.set()
await asyncio.gather(_stale_request(), _first_request(), _request_seeding_during_the_re_read())
assert redis.values[counter] == 6.0
class _HeldGetRedis(_SharedRedis):
"""Answers the first GET with the value its key had when it arrived, holding the reply until released."""
@ -2598,11 +2559,11 @@ async def test_a_cold_read_sent_before_an_add_started_does_not_join_it_when_the_
@pytest.mark.asyncio
async def test_a_re_read_redis_does_not_answer_stops_at_the_wait_budget_and_the_request_adds_its_cached_spend(
async def test_a_counter_lost_and_re_created_below_the_seed_gets_the_cached_spend_on_top(
monkeypatch: pytest.MonkeyPatch,
) -> None:
counter: Final = "spend:end_user:stalled-reread"
redis: Final = _HeldReadRedis()
counter: Final = "spend:end_user:re-created-below"
redis: Final = _SharedRedis()
_process(monkeypatch, redis, cached_spend=6.0)
read: Final = asyncio.Event()
seeded: Final = asyncio.Event()
@ -2612,8 +2573,36 @@ async def test_a_re_read_redis_does_not_answer_stops_at_the_wait_budget_and_the_
await _read_cold(counter)
read.set()
await seeded.wait()
del redis.values[counter]
redis.hold_reads = True
redis.values[counter] = 0.5
await ps.ensure_spend_counter_initialized(counter_key=counter, source_cache_key=_COLD_SEED_SOURCE)
assert await read_batched_spend_counter(counter) == (6.5, True)
async def _first_request() -> None:
await read.wait()
await _cold_start(redis, counter)
seeded.set()
await asyncio.gather(_stale_request(), _first_request())
assert redis.values[counter] == 6.5
@pytest.mark.asyncio
async def test_a_counter_holding_the_seed_is_left_alone_by_a_request_whose_source_cache_moved_past_it(
monkeypatch: pytest.MonkeyPatch,
) -> None:
counter: Final = "spend:end_user:source-moved"
redis: Final = _SharedRedis()
counters: Final = _process(monkeypatch, redis, cached_spend=6.0)
read: Final = asyncio.Event()
seeded: Final = asyncio.Event()
async def _stale_request() -> None:
with spend_counter_batch_scope(cast(RedisCache, redis), frozenset((counter,))):
await _read_cold(counter)
read.set()
await seeded.wait()
ps.user_api_key_cache.in_memory_cache.set_cache(key=_COLD_SEED_SOURCE, value={"spend": 6.25})
await ps.ensure_spend_counter_initialized(counter_key=counter, source_cache_key=_COLD_SEED_SOURCE)
async def _first_request() -> None:
@ -2621,10 +2610,153 @@ async def test_a_re_read_redis_does_not_answer_stops_at_the_wait_budget_and_the_
await _cold_start(redis, counter)
seeded.set()
await asyncio.wait_for(asyncio.gather(_stale_request(), _first_request()), timeout=2.0)
await asyncio.gather(_stale_request(), _first_request())
assert redis.values[counter] == 6.0
assert redis.reads_after_hold == 1
assert counters.in_memory_cache.get_cache(key=counter) == 6.0
@pytest.mark.asyncio
async def test_a_request_done_waiting_on_a_stalled_seed_adds_when_the_counter_holds_less_than_its_cached_spend(
monkeypatch: pytest.MonkeyPatch,
) -> None:
counter: Final = "spend:end_user:below-own-spend"
monkeypatch.setattr(spend_counter_reseed, "SPEND_COUNTER_COLD_SEED_JOIN_SECONDS", 0.01)
redis: Final = _HeldIncrementRedis(held_increments=1)
_process(monkeypatch, redis, cached_spend=6.0)
stalled: Final = asyncio.ensure_future(_cold_start(redis, counter))
await redis.arrived[0].wait()
with spend_counter_batch_scope(cast(RedisCache, redis), frozenset((counter,))):
await _read_cold(counter)
redis.values[counter] = 0.5
await ps.ensure_spend_counter_initialized(counter_key=counter, source_cache_key=_COLD_SEED_SOURCE)
while_stalled: Final = redis.values[counter]
redis.releases[0].set()
await stalled
assert while_stalled == 6.5
@pytest.mark.asyncio
async def test_a_counter_without_redis_set_only_if_missing_keeps_the_value_it_holds(
monkeypatch: pytest.MonkeyPatch,
) -> None:
counter: Final = "spend:end_user:no-redis-set-if-missing"
monkeypatch.setattr(ps, "spend_counter_cache", DualCache(in_memory_cache=InMemoryCache()))
first: Final = await ps.increment_spend_counter_cache(counter, 6.0, only_if_missing=True)
second: Final = await ps.increment_spend_counter_cache(counter, 4.0, only_if_missing=True)
added: Final = await ps.increment_spend_counter_cache(counter, 1.0)
assert (first, second, added) == (6.0, 6.0, 7.0)
@pytest.mark.asyncio
async def test_a_counter_set_by_a_request_done_waiting_counts_as_the_seed_for_a_later_request(
monkeypatch: pytest.MonkeyPatch,
) -> None:
counter: Final = "spend:end_user:set-counts-as-seed"
monkeypatch.setattr(spend_counter_reseed, "SPEND_COUNTER_COLD_SEED_JOIN_SECONDS", 0.01)
redis: Final = _HeldIncrementRedis(held_increments=1)
_process(monkeypatch, redis, cached_spend=6.0)
read: Final = asyncio.Event()
set_by_a_waiter: Final = asyncio.Event()
async def _stale_request() -> None:
with spend_counter_batch_scope(cast(RedisCache, redis), frozenset((counter,))):
await _read_cold(counter)
read.set()
await set_by_a_waiter.wait()
redis.values[counter] = 0.5
await ps.ensure_spend_counter_initialized(counter_key=counter, source_cache_key=_COLD_SEED_SOURCE)
stale: Final = asyncio.ensure_future(_stale_request())
await read.wait()
stalled: Final = asyncio.ensure_future(_cold_start(redis, counter))
await redis.arrived[0].wait()
await _cold_start(redis, counter)
set_by_a_waiter.set()
await stale
after_the_stale_request: Final = redis.values[counter]
redis.releases[0].set()
await stalled
assert after_the_stale_request == 6.5
class _CommandCountingRedis(_SharedRedis):
def __init__(self) -> None:
super().__init__()
self.commands: Final[list[str]] = [] # mutable-ok: the commands the test asserts on
async def async_batch_get_cache(self, key_list: list[str], **_: object) -> dict[str, float | None]:
self.commands.append("mget")
return await super().async_batch_get_cache(key_list)
async def async_get_cache(self, key: str, **_: object) -> float | None:
self.commands.append("get")
return await super().async_get_cache(key)
async def async_increment(self, key: str, value: float, **kwargs: bool) -> float:
self.commands.append("set_if_missing" if kwargs.get("only_if_missing") else "add")
return await super().async_increment(key, value, **kwargs)
@pytest.mark.asyncio
async def test_a_request_that_read_before_a_seed_landed_sends_one_write_and_keeps_its_value_for_later_reads(
monkeypatch: pytest.MonkeyPatch,
) -> None:
counter: Final = "spend:end_user:one-write"
redis: Final = _CommandCountingRedis()
_process(monkeypatch, redis, cached_spend=6.0)
read: Final = asyncio.Event()
seeded: Final = asyncio.Event()
async def _stale_request() -> None:
with spend_counter_batch_scope(cast(RedisCache, redis), frozenset((counter,))):
await _read_cold(counter)
read.set()
await seeded.wait()
redis.commands.clear()
await ps.ensure_spend_counter_initialized(counter_key=counter, source_cache_key=_COLD_SEED_SOURCE)
assert await read_batched_spend_counter(counter) == (6.0, True)
assert await ps.get_current_spend(counter_key=counter, fallback_spend=6.0) == 6.0
async def _first_request() -> None:
await read.wait()
await _cold_start(redis, counter)
seeded.set()
await asyncio.gather(_stale_request(), _first_request())
assert redis.commands == ["set_if_missing"]
@pytest.mark.asyncio
async def test_a_request_waits_on_other_requests_adds_for_the_limit_in_total_across_its_counters(
monkeypatch: pytest.MonkeyPatch,
) -> None:
limit: Final = 0.3
monkeypatch.setattr(spend_counter_reseed, "SPEND_COUNTER_COLD_SEED_JOIN_SECONDS", limit)
keys: Final = tuple(f"spend:end_user:shared-wait-{index}" for index in range(4))
redis: Final = _HeldIncrementRedis(held_increments=len(keys))
_process(monkeypatch, redis, cached_spend=6.0)
stalled: Final = [asyncio.ensure_future(_cold_start(redis, key)) for key in keys]
for arrived in redis.arrived:
await arrived.wait()
loop: Final = asyncio.get_running_loop()
started: Final = loop.time()
with spend_counter_batch_scope(cast(RedisCache, redis), frozenset(keys)):
for key in keys:
await ps.ensure_spend_counter_initialized(counter_key=key, source_cache_key=_COLD_SEED_SOURCE)
waited: Final = loop.time() - started
for release in redis.releases:
release.set()
await asyncio.gather(*stalled)
assert limit <= waited < 2 * limit
class _LostAndRecreatedAfterFailedMGetRedis(_SharedRedis):
@ -2657,9 +2789,9 @@ class _LostAndRecreatedAfterFailedMGetRedis(_SharedRedis):
self.cold_gets += 1
return await super().async_get_cache(key)
async def async_increment(self, key: str, value: float, **_: object) -> float:
async def async_increment(self, key: str, value: float, **kwargs: bool) -> float:
self._other_process_increments_after_the_cold_gets()
return await super().async_increment(key, value)
return await super().async_increment(key, value, **kwargs)
@pytest.mark.asyncio
@ -2686,8 +2818,8 @@ class _AppliedHeldThenResetRedis(_HeldReplyRedis):
super().__init__()
self.deleted = False
async def async_increment(self, key: str, value: float, **_: object) -> float:
await super().async_increment(key, value)
async def async_increment(self, key: str, value: float, **kwargs: bool) -> float:
await super().async_increment(key, value, **kwargs)
raise ConnectionError("connection reset")
async def async_delete_cache(self, key: str) -> None:
@ -2721,6 +2853,7 @@ async def test_requests_waiting_on_a_cold_seed_whose_request_was_cancelled_add_t
monkeypatch: pytest.MonkeyPatch,
) -> None:
counter: Final = "spend:end_user:cancelled-seed-with-waiters"
monkeypatch.setattr(spend_counter_reseed, "SPEND_COUNTER_COLD_SEED_JOIN_SECONDS", 60.0)
redis: Final = _StalledFirstIncrementRedis()
_process(monkeypatch, redis, cached_spend=6.0)
read: Final = asyncio.Event()
@ -2742,7 +2875,7 @@ async def test_requests_waiting_on_a_cold_seed_whose_request_was_cancelled_add_t
for _ in range(20):
await asyncio.sleep(0)
seed.cancel()
await asyncio.gather(joined_before_start, joined_after_start)
await asyncio.wait_for(asyncio.gather(joined_before_start, joined_after_start), timeout=5)
assert seed.cancelled()
assert redis.values[counter] == 6.0
@ -2766,13 +2899,13 @@ class _HeldIncrementRedis(_SharedRedis):
self.reads += 1
return await super().async_get_cache(key)
async def async_increment(self, key: str, value: float, **_: object) -> float:
async def async_increment(self, key: str, value: float, **kwargs: bool) -> float:
index: Final = self.increments
self.increments += 1
if index < len(self.arrived):
self.arrived[index].set()
await self.releases[index].wait()
return await super().async_increment(key, value)
return await super().async_increment(key, value, **kwargs)
@pytest.mark.asyncio
@ -2830,23 +2963,11 @@ async def test_a_request_that_read_the_counter_while_a_seed_was_in_flight_does_n
assert redis.values[counter] == 6.0
class _SlowGetRedis(_HeldIncrementRedis):
"""Also takes ``SECONDS`` to answer a GET."""
SECONDS: ClassVar[float] = 0.2
async def async_get_cache(self, key: str, **_: object) -> float | None:
await asyncio.sleep(self.SECONDS)
return await super().async_get_cache(key)
@pytest.mark.asyncio
async def test_a_seed_that_lands_late_in_the_wait_leaves_time_to_re_read_the_counter_instead_of_adding_again(
monkeypatch: pytest.MonkeyPatch,
) -> None:
async def test_a_seed_that_lands_late_in_the_wait_is_not_added_again(monkeypatch: pytest.MonkeyPatch) -> None:
counter: Final = "spend:end_user:late-landing-seed"
monkeypatch.setattr(spend_counter_reseed, "SPEND_COUNTER_COLD_SEED_JOIN_SECONDS", 0.5)
redis: Final = _SlowGetRedis(held_increments=1)
redis: Final = _HeldIncrementRedis(held_increments=1)
_process(monkeypatch, redis, cached_spend=6.0)
first: Final = asyncio.ensure_future(_cold_start(redis, counter))
@ -2859,6 +2980,35 @@ async def test_a_seed_that_lands_late_in_the_wait_leaves_time_to_re_read_the_cou
assert redis.values[counter] == 6.0
@pytest.mark.asyncio
async def test_a_request_joining_an_add_that_started_after_its_read_came_back_sends_no_write(
monkeypatch: pytest.MonkeyPatch,
) -> None:
counter: Final = "spend:end_user:joined-no-write"
monkeypatch.setattr(spend_counter_reseed, "SPEND_COUNTER_COLD_SEED_JOIN_SECONDS", 60.0)
redis: Final = _HeldIncrementRedis(held_increments=1)
_process(monkeypatch, redis, cached_spend=6.0)
read: Final = asyncio.Event()
async def _joining_request() -> None:
with spend_counter_batch_scope(cast(RedisCache, redis), frozenset((counter,))):
await _read_cold(counter)
read.set()
await redis.arrived[0].wait()
await ps.ensure_spend_counter_initialized(counter_key=counter, source_cache_key=_COLD_SEED_SOURCE)
joining: Final = asyncio.ensure_future(_joining_request())
await read.wait()
first: Final = asyncio.ensure_future(_cold_start(redis, counter))
await redis.arrived[0].wait()
for _ in range(100):
await asyncio.sleep(0)
redis.releases[0].set()
await asyncio.gather(joining, first)
assert (redis.values[counter], redis.increments) == (6.0, 1)
class _HeldSecondGetRedis(_HeldIncrementRedis):
"""Also answers the second GET with the value its key had when it arrived, holding the reply until released."""
@ -2917,7 +3067,63 @@ async def test_a_stalled_seed_finishing_does_not_hide_the_newer_seed_still_in_fl
late: Final = asyncio.ensure_future(_cold_start(redis, counter))
for _ in range(10):
await asyncio.sleep(0)
commands_while_the_newer_seed_was_in_flight: Final = redis.increments
redis.releases[1].set()
await asyncio.gather(newer, late)
assert commands_while_the_newer_seed_was_in_flight == 2
assert redis.values[counter] == 6.0
class _TargetRecordingRedis(_SharedRedis):
def __init__(self) -> None:
super().__init__()
self.targets: Final[list[str | None]] = [] # mutable-ok: the targets the test asserts on
async def async_get_cache(self, key: str, **_: object) -> float | None:
self.targets.append(current_service_target())
return 1.0
@pytest.mark.asyncio
async def test_the_cold_check_read_reports_under_spend_counters_inside_another_phases_target(
monkeypatch: pytest.MonkeyPatch,
) -> None:
redis: Final = _TargetRecordingRedis()
_process(monkeypatch, redis, cached_spend=6.0)
with service_target(AUTH_OBJECTS_TARGET):
await ps.ensure_spend_counter_initialized(counter_key="spend:key:target", source_cache_key=_COLD_SEED_SOURCE)
assert redis.targets == [SPEND_COUNTERS_TARGET]
@pytest.mark.asyncio
async def test_a_stale_read_on_a_real_client_seeds_a_missing_counter_with_its_ttl_and_keeps_an_existing_one(
monkeypatch: pytest.MonkeyPatch,
) -> None:
counter: Final = "spend:end_user:real-client"
client: Final = FakeAsyncRedis(server=FakeServer())
redis: Final = _RedisCacheOnClient(client)
redis.default_ttl = 120
_process(monkeypatch, redis, cached_spend=6.0)
read: Final = asyncio.Event()
seeded: Final = asyncio.Event()
async def _stale_request() -> None:
with spend_counter_batch_scope(redis, frozenset((counter,))):
await _read_cold(counter)
read.set()
await seeded.wait()
await client.delete(counter)
await ps.ensure_spend_counter_initialized(counter_key=counter, source_cache_key=_COLD_SEED_SOURCE)
async def _first_request() -> None:
await read.wait()
await _cold_start(redis, counter)
seeded.set()
await asyncio.gather(_stale_request(), _first_request())
assert float(await client.get(counter) or 0) == 6.0
assert 0 < await client.ttl(counter) <= 120