mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-20 10:02:54 +00:00
fix(proxy): read the seeded counter after waiting on its seed lock
A request that waited while another seeded a cold counter kept its batched miss and read the DB again; a caller that waited now re-reads Redis first
This commit is contained in:
parent
ee606be1dc
commit
1d571b71b5
2 changed files with 7 additions and 3 deletions
|
|
@ -218,14 +218,17 @@ class SpendCounterReseed:
|
|||
when the DB read succeeds, or None when the DB is unavailable.
|
||||
"""
|
||||
lock: Final = await SpendCounterReseed._get_lock(counter_key)
|
||||
waited_for_seed: Final = lock.locked()
|
||||
async with lock:
|
||||
batched: Final = await SpendCounterReseed._read_active_batch(counter_key)
|
||||
if batched is not None and batched[0] is not None:
|
||||
return batched[0]
|
||||
# Re-check after acquiring the lock. Skip in-memory on a clean
|
||||
# Redis miss - in-memory is per-pod-stale.
|
||||
# Redis miss - in-memory is per-pod-stale. The request's batched
|
||||
# read can predate the lock, so a caller that waited on it re-reads
|
||||
# Redis for the value the holder seeded.
|
||||
redis_clean_miss = batched is not None
|
||||
if spend_counter_cache.redis_cache is not None and not redis_clean_miss:
|
||||
if spend_counter_cache.redis_cache is not None and (waited_for_seed or not redis_clean_miss):
|
||||
try:
|
||||
val = await spend_counter_cache.redis_cache.async_get_cache(key=counter_key)
|
||||
if val is not None:
|
||||
|
|
|
|||
|
|
@ -1373,7 +1373,7 @@ class _FakeRedisMeetingAtTheWarmCheck(FakeAsyncRedis):
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_overlapping_cold_end_user_seeds_write_the_end_user_row_once(monkeypatch):
|
||||
async def test_overlapping_cold_end_user_seeds_read_and_write_the_end_user_row_once(monkeypatch: pytest.MonkeyPatch):
|
||||
counter_key: Final = "spend:end_user:customer-1"
|
||||
callers: Final = 3
|
||||
redis: Final = _FakeRedisMeetingAtTheWarmCheck(FakeServer(), callers)
|
||||
|
|
@ -1399,6 +1399,7 @@ async def test_overlapping_cold_end_user_seeds_write_the_end_user_row_once(monke
|
|||
await asyncio.wait_for(asyncio.gather(*(_request() for _ in range(callers))), timeout=5)
|
||||
|
||||
assert float(await redis.get(counter_key)) == 7.5
|
||||
prisma.db.litellm_endusertable.find_unique.assert_awaited_once()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue