From 2f4090725925a809d87039f0b91ce819295f6c88 Mon Sep 17 00:00:00 2001 From: yucheng Date: Sat, 31 Oct 2026 05:53:03 -0700 Subject: [PATCH 1/7] fix(proxy): add a process's cached spend to a cold spend counter once Overlapping cold starts of one spend counter inside one proxy process each added the process's whole cached spend, so N concurrent requests left N times the spend and refused a customer still under budget. Requests that read the counter cold before this process's own add landed now join that add, or re-read the counter and add again only when it is gone. Other processes still add their own cached spend as before. Refs #47062 --- litellm/proxy/db/spend_counter_reseed.py | 51 ++++- litellm/proxy/proxy_server.py | 10 +- .../spend_tracking/spend_counter_batch.py | 26 ++- .../spend/test_end_user_budget_enforcement.py | 135 ++++++++++++- .../test_spend_counter_cold_seed.py | 179 ++++++++++++++++++ 5 files changed, 397 insertions(+), 4 deletions(-) create mode 100644 tests/unit/proxy/proxy_server/test_spend_counter_cold_seed.py diff --git a/litellm/proxy/db/spend_counter_reseed.py b/litellm/proxy/db/spend_counter_reseed.py index 721999b1868..cc81530a2f8 100644 --- a/litellm/proxy/db/spend_counter_reseed.py +++ b/litellm/proxy/db/spend_counter_reseed.py @@ -13,8 +13,10 @@ memory in long-lived deployments. """ import asyncio +import functools from collections import OrderedDict -from collections.abc import Mapping +from collections.abc import Awaitable, Callable, Mapping +from dataclasses import dataclass from datetime import datetime, timezone from types import MappingProxyType from typing import TYPE_CHECKING, ClassVar, Final, Optional @@ -27,8 +29,10 @@ from litellm.proxy._types import Litellm_EntityType from litellm.proxy.db.db_lookup_gate import bounded_db_lookup, db_lookup_gate from litellm.proxy.spend_tracking.spend_counter_batch import ( SPEND_COUNTERS_TARGET, + forget_spend_counter, read_batched_spend_counter, record_spend_counter_value, + spend_counter_tick, ) from litellm.repositories.organization_repository import OrganizationRepository from litellm.repositories.project_repository import ProjectRepository @@ -68,6 +72,30 @@ _WINDOW_SPEND_LOG_FIELDS: Final[Mapping[str, str]] = MappingProxyType( ) +@dataclass(slots=True) +class _CachedSpendSeed: + """This process adding its cached spend to one cold counter: the add in flight, and the tick the last one landed.""" + + landed_at: int = 0 + in_flight: asyncio.Task[float | None] | None = None + + def start(self, add: Callable[[], Awaitable[float | None]]) -> asyncio.Task[float | None]: + in_flight: Final = asyncio.ensure_future(add()) + in_flight.add_done_callback(self._settle) + self.in_flight = in_flight + return in_flight + + def _settle(self, done: asyncio.Future[float | None]) -> None: + self.in_flight = None + if not done.cancelled() and done.exception() is None: + self.landed_at = spend_counter_tick() + + +@functools.lru_cache(maxsize=SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE) +def _cached_spend_seed(spend_counter_cache: "DualCache", counter_key: str) -> _CachedSpendSeed: + return _CachedSpendSeed() + + def _as_utc(value: datetime) -> datetime: return value if value.tzinfo is not None else value.replace(tzinfo=timezone.utc) @@ -112,6 +140,27 @@ class SpendCounterReseed: SpendCounterReseed._locks.popitem(last=False) return lock + @staticmethod + async def add_cached_spend_once( + spend_counter_cache: "DualCache", + counter_key: str, + read_at: int, + add: Callable[[], Awaitable[float | None]], + is_warm: Callable[[], Awaitable[bool]], + ) -> None: + """Run ``add``, this process adding its cached spend to a cold counter, once for all of this process's + requests that read the counter cold (at or after tick ``read_at``) before that add landed. A request joins the + add in flight and gets its result or error. When the add already landed, the request re-reads the counter and + adds again only if the counter is gone. Each other process still adds its own cached spend.""" + seed: Final = _cached_spend_seed(spend_counter_cache, counter_key) + if seed.in_flight is None and seed.landed_at > read_at: + forget_spend_counter(counter_key) + if await is_warm(): + return + added: Final = await asyncio.shield(seed.in_flight or seed.start(add)) + if added is not None: + record_spend_counter_value(counter_key, added) + @staticmethod @with_service_target(SPEND_COUNTERS_TARGET) async def increment_in_memory(spend_counter_cache: "DualCache", counter_key: str, increment: float) -> float | None: diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index a767f5423ed..b43733286cd 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -936,6 +936,7 @@ from litellm.proxy.spend_tracking.spend_counter_batch import ( read_batched_spend_counter, record_spend_counter_value, spend_counter_batch_scope, + spend_counter_read_tick, ) from litellm.proxy.spend_tracking.spend_management_endpoints import ( router as spend_management_router, @@ -4085,6 +4086,7 @@ async def ensure_spend_counter_initialized( counter_key: str, source_cache_key: str | list[str], ) -> None: + read_at: Final = spend_counter_read_tick() is_warm: Final = await _is_spend_counter_cache_warm(counter_key=counter_key) if is_warm is False: # Shares the per-counter lock with get_current_spend. @@ -4098,7 +4100,13 @@ async def ensure_spend_counter_initialized( # DB unavailable - fall back to in-process cache (may be stale). base_spend: Final = await _get_source_cache_base_spend(source_cache_key=source_cache_key) if base_spend > 0: - await increment_spend_counter_cache(counter_key=counter_key, increment=base_spend) + await SpendCounterReseed.add_cached_spend_once( + spend_counter_cache=spend_counter_cache, + counter_key=counter_key, + read_at=read_at, + add=lambda: increment_spend_counter_cache(counter_key=counter_key, increment=base_spend), + is_warm=lambda: _is_spend_counter_cache_warm(counter_key=counter_key), + ) _ensure_spend_counter_initialized: Final = ensure_spend_counter_initialized diff --git a/litellm/proxy/spend_tracking/spend_counter_batch.py b/litellm/proxy/spend_tracking/spend_counter_batch.py index 7e0821e091e..49280448f75 100644 --- a/litellm/proxy/spend_tracking/spend_counter_batch.py +++ b/litellm/proxy/spend_tracking/spend_counter_batch.py @@ -1,6 +1,7 @@ """One Redis MGET per phase (admission, reservation, post-call) for the spend counters it reads, not one GET each.""" import asyncio +import itertools from collections.abc import Iterator, Mapping, Sequence from contextvars import ContextVar, Token from dataclasses import dataclass @@ -22,6 +23,12 @@ from litellm.proxy.common_utils.user_api_key_cache import ( _CounterValues: Final = TypeAdapter(dict[str, float | None]) _NO_VALUES: Final[Mapping[str, float | None]] = MappingProxyType({}) SPEND_COUNTERS_TARGET: Final = "spend_counters" +_ticks: Final = itertools.count(1) + + +def spend_counter_tick() -> int: + """The next number in a per-process sequence that orders spend counter reads and cold seeds.""" + return next(_ticks) @dataclass(frozen=True, slots=True) @@ -39,10 +46,21 @@ class SpendCounterBatch: hook declares whatever is bound but unread, so whoever flushes first (the auth object prefetch, usually) carries the spend counters in the same round trip.""" - __slots__ = ("_fetched", "_inflight", "_keys", "_loaded", "_lock", "_open", "_redis_cache", "_request_batch") + __slots__ = ( + "_fetched", + "_inflight", + "_keys", + "_loaded", + "_lock", + "_open", + "_redis_cache", + "_request_batch", + "opened_at", + ) def __init__(self, redis_cache: RedisCache) -> None: self._redis_cache: Final = redis_cache + self.opened_at: Final = spend_counter_tick() self._lock: Final = asyncio.Lock() self._open = True self._keys: frozenset[str] = frozenset() @@ -145,6 +163,12 @@ def active_spend_counter_batch() -> SpendCounterBatch | None: return _active_batch.get() +def spend_counter_read_tick() -> int: + """A tick no later than this context's next spend counter read: the open batch's opening tick, else a new one.""" + batch: Final = _active_batch.get() + return batch.opened_at if batch is not None and batch.is_open else spend_counter_tick() + + class spend_counter_batch_scope: """Reads inside the scope share one MGET for the keys bound here or by ``bind_*`` calls inside it. Opened inside a scope whose batch is still open, it binds into that batch so both phases share the MGET.""" diff --git a/tests/integration/spend/test_end_user_budget_enforcement.py b/tests/integration/spend/test_end_user_budget_enforcement.py index 02c8aee8f5e..9ba53f133ec 100644 --- a/tests/integration/spend/test_end_user_budget_enforcement.py +++ b/tests/integration/spend/test_end_user_budget_enforcement.py @@ -1,17 +1,20 @@ import json +import os import re import uuid from collections.abc import Callable, Mapping +from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass from typing import Final import httpx import pytest from integration._support.client import Gateway, Scenario, eventually, object_value, string_value -from integration._support.database import read_rows +from integration._support.database import read_rows, write_rows from integration._support.upstream import delete_scenario, register_scenario from integration.cost_calculation.cost_tracking_case import JsonResponse from pydantic import BaseModel, JsonValue +from redis import Redis PROVIDER_AUTHORIZATION: Final = "Bearer integration-provider-key" CANNED_CONTENT: Final = "Hello! This is a mock response from the fake OpenAI endpoint." @@ -639,3 +642,133 @@ def test_streamed_call_is_refused_as_json_once_over_budget(gateway: Gateway) -> assert "data:" not in denied.text, denied.text _assert_budget_refusal(denied, end_user, 0.05) assert _observed_requests(upstream) == [] + + +_COLD_BURST: Final = 40 +_COLD_SEED_SPEND: Final = 6.0 +_COLD_BUDGET: Final = 10.0 +_CHAT_COST: Final = 20 * 0.001 + 20 * 0.002 + + +def _serves_model(gateway: Gateway, model: str) -> bool: + response: Final = gateway.request("GET", "/model/info") + return response.status_code == 200 and any( + object_value(entry)["model_name"] == model for entry in object_value(response.json())["data"] + ) + + +def _counter_cache() -> Redis: + return Redis(host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"])) + + +def _counter(cache: Redis, end_user: str) -> float | None: + raw: Final = cache.get(f"spend:end_user:{end_user}") + return None if raw is None else float(raw) + + +def _lose_counter(cache: Redis, end_user: str) -> None: + cache.delete(f"spend:end_user:{end_user}") + + +def _set_end_user_row_spend(end_user: str, spend: float) -> None: + write_rows('UPDATE "LiteLLM_EndUserTable" SET spend = %s WHERE user_id = %s', (str(spend), end_user)) + + +def _await_row_spend(end_user: str, spend: float) -> None: + eventually( + lambda: _end_user_row(end_user), + lambda rows: len(rows) == 1 and float(str(rows[0]["spend"])) == pytest.approx(spend), + seconds=70, + ) + + +def _refused_spend(denied: httpx.Response, end_user: str) -> float: + error: Final = _denial_error(denied) + assert error["type"] == "budget_exceeded", denied.text + matched: Final = _REFUSAL_MESSAGE.match(string_value(error["message"])) + assert matched is not None, denied.text + assert matched.group("user") == end_user + return float(matched.group("spend")) + + +@pytest.mark.parametrize("spelling", ("user", "responses_user", "messages_metadata_user_id")) +def test_overlapping_cold_starts_count_the_end_user_spend_once(gateway: Gateway, spelling: str) -> None: + chosen: Final = SPELLINGS[spelling] + with ( + gateway.scenario() as scenario, + httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, + _counter_cache() as cache, + ): + deployment: Final = _deployment(scenario, chosen) + key: Final = scenario.key(models=[deployment.model]) + end_user: Final = f"integration-end-user-{uuid.uuid4().hex}" + _new_customer(gateway, end_user, max_budget=_COLD_BUDGET, spend=_COLD_SEED_SPEND) + scenario.cleanups.callback(gateway.request, "POST", "/customer/delete", {"user_ids": [end_user]}) + assert _customer_info(gateway, end_user).spend == _COLD_SEED_SPEND + assert _counter(cache, end_user) is None + upstream.delete("/__observations").raise_for_status() + + with ThreadPoolExecutor(max_workers=_COLD_BURST) as pool: + burst: Final = tuple( + pool.map( + lambda index: _send(gateway, chosen, deployment, key, end_user, f"cold {index}"), + range(_COLD_BURST), + ) + ) + refused: Final = [response.text for response in burst if response.status_code != 200] + assert refused == [], refused + follow_up: Final = _send(gateway, chosen, deployment, key, end_user, "cold follow-up") + _assert_served(chosen, follow_up) + assert len(_observed_requests(upstream)) == _COLD_BURST + 1 + + expected_spend: Final = _COLD_SEED_SPEND + (_COLD_BURST + 1) * _CHAT_COST + _await_row_spend(end_user, expected_spend) + assert _customer_info(gateway, end_user).spend == pytest.approx(expected_spend) + counted: Final = eventually( + lambda: _counter(cache, end_user), + lambda value: value is not None and value == pytest.approx(expected_spend), + seconds=30, + return_last_on_timeout=True, + ) + assert counted == pytest.approx(expected_spend) + + +def test_a_lost_counter_keeps_spend_this_proxy_has_not_written_yet(gateway: Gateway) -> None: + with gateway.scenario() as scenario, _counter_cache() as cache: + model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.25) + key: Final = scenario.key(models=[model]) + end_user: Final = f"integration-end-user-{uuid.uuid4().hex}" + call_cost: Final = 20 * 0.001 + 20 * 0.25 + _new_customer(gateway, end_user, max_budget=_COLD_BUDGET, spend=_COLD_SEED_SPEND) + scenario.cleanups.callback(gateway.request, "POST", "/customer/delete", {"user_ids": [end_user]}) + served: Final = _chat(gateway, model, key, end_user) + assert served.status_code == 200, served.text + _await_row_spend(end_user, _COLD_SEED_SPEND + call_cost) + _set_end_user_row_spend(end_user, _COLD_SEED_SPEND) + _lose_counter(cache, end_user) + + denied: Final = _chat(gateway, model, key, end_user) + + assert _refused_spend(denied, end_user) == pytest.approx(_COLD_SEED_SPEND + call_cost) + + +def test_a_lost_counter_keeps_spend_another_proxy_has_not_written_yet(gateway: Gateway, peer: Gateway) -> None: + with gateway.scenario() as scenario, _counter_cache() as cache: + model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.149) + eventually(lambda: _serves_model(peer, model), bool, seconds=30) + key: Final = scenario.key(models=[model]) + end_user: Final = f"integration-end-user-{uuid.uuid4().hex}" + call_cost: Final = 20 * 0.001 + 20 * 0.149 + _new_customer(gateway, end_user, max_budget=_COLD_BUDGET, spend=_COLD_SEED_SPEND) + scenario.cleanups.callback(gateway.request, "POST", "/customer/delete", {"user_ids": [end_user]}) + first: Final = _chat(peer, model, key, end_user) + assert first.status_code == 200, first.text + _await_row_spend(end_user, _COLD_SEED_SPEND + call_cost) + _set_end_user_row_spend(end_user, _COLD_SEED_SPEND) + _lose_counter(cache, end_user) + second: Final = _chat(peer, model, key, end_user) + assert second.status_code == 200, second.text + + denied: Final = _chat(gateway, model, key, end_user) + + assert _refused_spend(denied, end_user) == pytest.approx(_COLD_SEED_SPEND + 2 * call_cost) diff --git a/tests/unit/proxy/proxy_server/test_spend_counter_cold_seed.py b/tests/unit/proxy/proxy_server/test_spend_counter_cold_seed.py new file mode 100644 index 00000000000..d51c36a9c64 --- /dev/null +++ b/tests/unit/proxy/proxy_server/test_spend_counter_cold_seed.py @@ -0,0 +1,179 @@ +"""Overlapping cold starts of one spend counter inside one process add that process's cached spend once.""" + +import asyncio +from typing import Final, cast + +import pytest + +import litellm.proxy.proxy_server as ps +from litellm.caching.dual_cache import DualCache +from litellm.caching.in_memory_cache import InMemoryCache +from litellm.caching.redis_cache import RedisCache +from litellm.proxy.spend_tracking.spend_counter_batch import active_spend_counter_batch, spend_counter_batch_scope + +SOURCE: Final = "end_user_id:customer-1" + + +class _SharedRedis: + """The counter commands a cold start sends, yielding like a network call so overlapping requests interleave.""" + + def __init__(self) -> None: + self.values: Final[dict[str, float]] = {} # mutable-ok: stands in for the Redis keyspace + self.failing = False + + async def async_batch_get_cache(self, key_list: list[str], **_: object) -> dict[str, float | None]: + await asyncio.sleep(0) + return {key: self.values.get(key) for key in key_list} + + async def async_get_cache(self, key: str, **_: object) -> float | None: + await asyncio.sleep(0) + return self.values.get(key) + + async def async_increment(self, key: str, value: float, **_: object) -> float: + await asyncio.sleep(0) + if self.failing: + raise ConnectionError("redis down") + self.values[key] = self.values.get(key, 0.0) + value + return self.values[key] + + async def async_delete_cache(self, key: str) -> None: + await asyncio.sleep(0) + if not self.failing: + self.values.pop(key, None) + + +def _process(monkeypatch: pytest.MonkeyPatch, redis: _SharedRedis | None, cached_spend: float) -> DualCache: + counters: Final = DualCache(in_memory_cache=InMemoryCache(), redis_cache=cast(RedisCache, redis)) + objects: Final = DualCache(in_memory_cache=InMemoryCache()) + objects.in_memory_cache.set_cache(key=SOURCE, value={"spend": cached_spend}) + monkeypatch.setattr(ps, "spend_counter_cache", counters) + monkeypatch.setattr(ps, "user_api_key_cache", objects) + monkeypatch.setattr(ps, "prisma_client", object()) + return counters + + +async def _cold_start(redis: _SharedRedis | None, counter: str) -> None: + with spend_counter_batch_scope(cast(RedisCache, redis) if redis is not None else None, frozenset((counter,))): + await ps.ensure_spend_counter_initialized(counter_key=counter, source_cache_key=SOURCE) + + +async def _read_cold(counter: str) -> None: + batch: Final = active_spend_counter_batch() + assert batch is not None + assert await batch.read(counter) == (None, True) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("with_redis", (True, False)) +async def test_overlapping_cold_starts_in_one_process_add_its_cached_spend_once( + monkeypatch: pytest.MonkeyPatch, with_redis: bool +) -> None: + counter: Final = f"spend:end_user:once-{with_redis}" + redis: Final = _SharedRedis() if with_redis else None + counters: Final = _process(monkeypatch, redis, cached_spend=6.0) + + await asyncio.gather(*(_cold_start(redis, counter) for _ in range(5))) + + seeded: Final = redis.values.get(counter) if redis is not None else counters.in_memory_cache.get_cache(key=counter) + assert seeded == 6.0 + + +@pytest.mark.asyncio +async def test_a_request_that_read_the_counter_before_this_process_seeded_it_does_not_add_again( + monkeypatch: pytest.MonkeyPatch, +) -> None: + counter: Final = "spend:end_user:stale-reader" + redis: Final = _SharedRedis() + _process(monkeypatch, redis, cached_spend=6.0) + read: Final = asyncio.Event() + + async def _stale_request() -> None: + with spend_counter_batch_scope(cast(RedisCache, redis), frozenset((counter,))): + await _read_cold(counter) + read.set() + await seeded.wait() + await ps.ensure_spend_counter_initialized(counter_key=counter, source_cache_key=SOURCE) + + seeded: Final = asyncio.Event() + + async def _first_request() -> None: + await read.wait() + await _cold_start(redis, counter) + seeded.set() + + await asyncio.gather(_stale_request(), _first_request()) + + assert redis.values[counter] == 6.0 + + +@pytest.mark.asyncio +async def test_a_counter_lost_again_after_this_process_seeded_it_is_seeded_again( + monkeypatch: pytest.MonkeyPatch, +) -> None: + counter: Final = "spend:end_user:lost-again" + redis: Final = _SharedRedis() + _process(monkeypatch, redis, cached_spend=6.0) + read: Final = asyncio.Event() + seeded: Final = asyncio.Event() + + async def _stale_request() -> None: + with spend_counter_batch_scope(cast(RedisCache, redis), frozenset((counter,))): + await _read_cold(counter) + read.set() + await seeded.wait() + del redis.values[counter] + await ps.ensure_spend_counter_initialized(counter_key=counter, source_cache_key=SOURCE) + + async def _first_request() -> None: + await read.wait() + await _cold_start(redis, counter) + seeded.set() + + await asyncio.gather(_stale_request(), _first_request()) + + assert redis.values[counter] == 6.0 + + +@pytest.mark.asyncio +async def test_another_process_seeding_after_this_request_read_the_counter_still_sums( + monkeypatch: pytest.MonkeyPatch, +) -> None: + counter: Final = "spend:end_user:two-processes" + redis: Final = _SharedRedis() + read: Final = asyncio.Event() + a_seeded: Final = asyncio.Event() + + async def _process_b() -> None: + with spend_counter_batch_scope(cast(RedisCache, redis), frozenset((counter,))): + await _read_cold(counter) + read.set() + await a_seeded.wait() + _process(monkeypatch, redis, cached_spend=5.0) + await ps.ensure_spend_counter_initialized(counter_key=counter, source_cache_key=SOURCE) + + async def _process_a() -> None: + await read.wait() + _process(monkeypatch, redis, cached_spend=3.0) + await _cold_start(redis, counter) + a_seeded.set() + + await asyncio.gather(_process_b(), _process_a()) + + assert redis.values[counter] == 8.0 + + +@pytest.mark.asyncio +async def test_a_failed_seed_fails_the_requests_that_joined_it_and_the_next_cold_start_seeds( + monkeypatch: pytest.MonkeyPatch, +) -> None: + counter: Final = "spend:end_user:failed-seed" + redis: Final = _SharedRedis() + _process(monkeypatch, redis, cached_spend=6.0) + redis.failing = True + + outcomes: Final = await asyncio.gather(*(_cold_start(redis, counter) for _ in range(3)), return_exceptions=True) + redis.failing = False + await _cold_start(redis, counter) + + assert [type(outcome) for outcome in outcomes] == [ConnectionError] * 3 + assert redis.values[counter] == 6.0 From 5a031c45b5a3a5439cc89524ac5efdc92deb5bd6 Mon Sep 17 00:00:00 2001 From: yucheng Date: Sat, 31 Oct 2026 10:08:28 -0700 Subject: [PATCH 2/7] fix(proxy): re-read a cold counter after an earlier seed lands, and seed on its own from another event loop A request that read the counter cold after this process's add started now waits for that add and re-reads the counter, so a counter deleted before the add returned is seeded again. A request on another event loop than the add in flight adds on its own, as before. --- litellm/proxy/db/spend_counter_reseed.py | 21 ++++-- .../test_spend_counter_cold_seed.py | 65 +++++++++++++++++++ 2 files changed, 81 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/db/spend_counter_reseed.py b/litellm/proxy/db/spend_counter_reseed.py index cc81530a2f8..1d9eb5f95b2 100644 --- a/litellm/proxy/db/spend_counter_reseed.py +++ b/litellm/proxy/db/spend_counter_reseed.py @@ -74,12 +74,18 @@ _WINDOW_SPEND_LOG_FIELDS: Final[Mapping[str, str]] = MappingProxyType( @dataclass(slots=True) class _CachedSpendSeed: - """This process adding its cached spend to one cold counter: the add in flight, and the tick the last one landed.""" + """This process adding its cached spend to one cold counter: the add in flight, the tick it started at, and the + tick the last one landed at.""" + started_at: int = 0 landed_at: int = 0 in_flight: asyncio.Task[float | None] | None = None + def on_this_loop(self) -> bool: + return self.in_flight is None or self.in_flight.get_loop() is asyncio.get_running_loop() + def start(self, add: Callable[[], Awaitable[float | None]]) -> asyncio.Task[float | None]: + self.started_at = spend_counter_tick() in_flight: Final = asyncio.ensure_future(add()) in_flight.add_done_callback(self._settle) self.in_flight = in_flight @@ -149,15 +155,20 @@ class SpendCounterReseed: is_warm: Callable[[], Awaitable[bool]], ) -> None: """Run ``add``, this process adding its cached spend to a cold counter, once for all of this process's - requests that read the counter cold (at or after tick ``read_at``) before that add landed. A request joins the - add in flight and gets its result or error. When the add already landed, the request re-reads the counter and - adds again only if the counter is gone. Each other process still adds its own cached spend.""" + requests that read the counter cold (at or after tick ``read_at``) before that add landed. A request joins an + add that started after its read and gets its result or error. When an add landed after its read, or one that + started before its read finishes, the request re-reads the counter and adds again only if the counter is gone. + A request on another event loop than the add in flight, and each other process, still adds its own cached + spend.""" seed: Final = _cached_spend_seed(spend_counter_cache, counter_key) + earlier: Final = seed.in_flight + if earlier is not None and seed.started_at < read_at and seed.on_this_loop(): + await asyncio.wait((earlier,)) if seed.in_flight is None and seed.landed_at > read_at: forget_spend_counter(counter_key) if await is_warm(): return - added: Final = await asyncio.shield(seed.in_flight or seed.start(add)) + added: Final = await (asyncio.shield(seed.in_flight or seed.start(add)) if seed.on_this_loop() else add()) if added is not None: record_spend_counter_value(counter_key, added) diff --git a/tests/unit/proxy/proxy_server/test_spend_counter_cold_seed.py b/tests/unit/proxy/proxy_server/test_spend_counter_cold_seed.py index d51c36a9c64..2662518e941 100644 --- a/tests/unit/proxy/proxy_server/test_spend_counter_cold_seed.py +++ b/tests/unit/proxy/proxy_server/test_spend_counter_cold_seed.py @@ -1,6 +1,7 @@ """Overlapping cold starts of one spend counter inside one process add that process's cached spend once.""" import asyncio +import threading from typing import Final, cast import pytest @@ -177,3 +178,67 @@ async def test_a_failed_seed_fails_the_requests_that_joined_it_and_the_next_cold assert [type(outcome) for outcome in outcomes] == [ConnectionError] * 3 assert redis.values[counter] == 6.0 + + +class _HeldReplyRedis(_SharedRedis): + """Applies an increment at once but holds its reply until released, like a slow network round trip.""" + + def __init__(self) -> None: + super().__init__() + self.applied: Final = asyncio.Event() + self.release: Final = asyncio.Event() + + async def async_increment(self, key: str, value: float, **_: object) -> float: + self.values[key] = self.values.get(key, 0.0) + value + total: Final = self.values[key] + self.applied.set() + await self.release.wait() + return total + + +@pytest.mark.asyncio +async def test_a_counter_deleted_before_an_earlier_add_returns_is_seeded_by_a_later_cold_reader( + monkeypatch: pytest.MonkeyPatch, +) -> None: + counter: Final = "spend:end_user:deleted-mid-add" + redis: Final = _HeldReplyRedis() + _process(monkeypatch, redis, cached_spend=6.0) + + first: Final = asyncio.ensure_future(_cold_start(redis, counter)) + await redis.applied.wait() + del redis.values[counter] + later: Final = asyncio.ensure_future(_cold_start(redis, counter)) + for _ in range(20): + await asyncio.sleep(0) + redis.release.set() + await asyncio.gather(first, later) + + assert redis.values[counter] == 6.0 + + +def test_a_cold_start_on_another_event_loop_adds_on_its_own(monkeypatch: pytest.MonkeyPatch) -> None: + counter: Final = "spend:end_user:two-loops" + held: Final = threading.Event() + release: Final = threading.Event() + + class _ThreadHeldRedis(_SharedRedis): + async def async_increment(self, key: str, value: float, **_: object) -> float: + if threading.current_thread() is not threading.main_thread(): + held.set() + while not release.is_set(): + await asyncio.sleep(0.001) + return await super().async_increment(key, value) + + redis: Final = _ThreadHeldRedis() + _process(monkeypatch, redis, cached_spend=6.0) + other_loop: Final = threading.Thread(target=lambda: asyncio.run(_cold_start(redis, counter)), daemon=True) + other_loop.start() + assert held.wait(5) + + try: + asyncio.run(_cold_start(redis, counter)) + finally: + release.set() + other_loop.join(5) + + assert redis.values[counter] == 12.0 From 5264872c2016a58085253ae8709588078fe615b4 Mon Sep 17 00:00:00 2001 From: yucheng Date: Sat, 31 Oct 2026 10:20:41 -0700 Subject: [PATCH 3/7] test(proxy): read the counter directly in the another-proxy lost-counter test The refusal there can come from budget reservation, whose message carries the request's own reservation, so the test now checks the counter in Redis. --- .../spend/test_end_user_budget_enforcement.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/integration/spend/test_end_user_budget_enforcement.py b/tests/integration/spend/test_end_user_budget_enforcement.py index 9ba53f133ec..6c129015494 100644 --- a/tests/integration/spend/test_end_user_budget_enforcement.py +++ b/tests/integration/spend/test_end_user_budget_enforcement.py @@ -768,7 +768,15 @@ def test_a_lost_counter_keeps_spend_another_proxy_has_not_written_yet(gateway: G _lose_counter(cache, end_user) second: Final = _chat(peer, model, key, end_user) assert second.status_code == 200, second.text + expected_spend: Final = _COLD_SEED_SPEND + 2 * call_cost + counted: Final = eventually( + lambda: _counter(cache, end_user), + lambda value: value is not None and value == pytest.approx(expected_spend), + seconds=30, + return_last_on_timeout=True, + ) denied: Final = _chat(gateway, model, key, end_user) - assert _refused_spend(denied, end_user) == pytest.approx(_COLD_SEED_SPEND + 2 * call_cost) + assert counted == pytest.approx(expected_spend) + assert _denial_error(denied)["type"] == "budget_exceeded", denied.text From 8a6d572b59164b1e5167a356cdb40ba360289ea0 Mon Sep 17 00:00:00 2001 From: yucheng Date: Sat, 31 Oct 2026 10:24:54 -0700 Subject: [PATCH 4/7] test(proxy): move the cold seed tests into test_spend_counters.py --- .../test_spend_counter_cold_seed.py | 244 ----------------- .../proxy/proxy_server/test_spend_counters.py | 247 +++++++++++++++++- 2 files changed, 242 insertions(+), 249 deletions(-) delete mode 100644 tests/unit/proxy/proxy_server/test_spend_counter_cold_seed.py diff --git a/tests/unit/proxy/proxy_server/test_spend_counter_cold_seed.py b/tests/unit/proxy/proxy_server/test_spend_counter_cold_seed.py deleted file mode 100644 index 2662518e941..00000000000 --- a/tests/unit/proxy/proxy_server/test_spend_counter_cold_seed.py +++ /dev/null @@ -1,244 +0,0 @@ -"""Overlapping cold starts of one spend counter inside one process add that process's cached spend once.""" - -import asyncio -import threading -from typing import Final, cast - -import pytest - -import litellm.proxy.proxy_server as ps -from litellm.caching.dual_cache import DualCache -from litellm.caching.in_memory_cache import InMemoryCache -from litellm.caching.redis_cache import RedisCache -from litellm.proxy.spend_tracking.spend_counter_batch import active_spend_counter_batch, spend_counter_batch_scope - -SOURCE: Final = "end_user_id:customer-1" - - -class _SharedRedis: - """The counter commands a cold start sends, yielding like a network call so overlapping requests interleave.""" - - def __init__(self) -> None: - self.values: Final[dict[str, float]] = {} # mutable-ok: stands in for the Redis keyspace - self.failing = False - - async def async_batch_get_cache(self, key_list: list[str], **_: object) -> dict[str, float | None]: - await asyncio.sleep(0) - return {key: self.values.get(key) for key in key_list} - - async def async_get_cache(self, key: str, **_: object) -> float | None: - await asyncio.sleep(0) - return self.values.get(key) - - async def async_increment(self, key: str, value: float, **_: object) -> float: - await asyncio.sleep(0) - if self.failing: - raise ConnectionError("redis down") - self.values[key] = self.values.get(key, 0.0) + value - return self.values[key] - - async def async_delete_cache(self, key: str) -> None: - await asyncio.sleep(0) - if not self.failing: - self.values.pop(key, None) - - -def _process(monkeypatch: pytest.MonkeyPatch, redis: _SharedRedis | None, cached_spend: float) -> DualCache: - counters: Final = DualCache(in_memory_cache=InMemoryCache(), redis_cache=cast(RedisCache, redis)) - objects: Final = DualCache(in_memory_cache=InMemoryCache()) - objects.in_memory_cache.set_cache(key=SOURCE, value={"spend": cached_spend}) - monkeypatch.setattr(ps, "spend_counter_cache", counters) - monkeypatch.setattr(ps, "user_api_key_cache", objects) - monkeypatch.setattr(ps, "prisma_client", object()) - return counters - - -async def _cold_start(redis: _SharedRedis | None, counter: str) -> None: - with spend_counter_batch_scope(cast(RedisCache, redis) if redis is not None else None, frozenset((counter,))): - await ps.ensure_spend_counter_initialized(counter_key=counter, source_cache_key=SOURCE) - - -async def _read_cold(counter: str) -> None: - batch: Final = active_spend_counter_batch() - assert batch is not None - assert await batch.read(counter) == (None, True) - - -@pytest.mark.asyncio -@pytest.mark.parametrize("with_redis", (True, False)) -async def test_overlapping_cold_starts_in_one_process_add_its_cached_spend_once( - monkeypatch: pytest.MonkeyPatch, with_redis: bool -) -> None: - counter: Final = f"spend:end_user:once-{with_redis}" - redis: Final = _SharedRedis() if with_redis else None - counters: Final = _process(monkeypatch, redis, cached_spend=6.0) - - await asyncio.gather(*(_cold_start(redis, counter) for _ in range(5))) - - seeded: Final = redis.values.get(counter) if redis is not None else counters.in_memory_cache.get_cache(key=counter) - assert seeded == 6.0 - - -@pytest.mark.asyncio -async def test_a_request_that_read_the_counter_before_this_process_seeded_it_does_not_add_again( - monkeypatch: pytest.MonkeyPatch, -) -> None: - counter: Final = "spend:end_user:stale-reader" - redis: Final = _SharedRedis() - _process(monkeypatch, redis, cached_spend=6.0) - read: Final = asyncio.Event() - - async def _stale_request() -> None: - with spend_counter_batch_scope(cast(RedisCache, redis), frozenset((counter,))): - await _read_cold(counter) - read.set() - await seeded.wait() - await ps.ensure_spend_counter_initialized(counter_key=counter, source_cache_key=SOURCE) - - seeded: Final = asyncio.Event() - - async def _first_request() -> None: - await read.wait() - await _cold_start(redis, counter) - seeded.set() - - await asyncio.gather(_stale_request(), _first_request()) - - assert redis.values[counter] == 6.0 - - -@pytest.mark.asyncio -async def test_a_counter_lost_again_after_this_process_seeded_it_is_seeded_again( - monkeypatch: pytest.MonkeyPatch, -) -> None: - counter: Final = "spend:end_user:lost-again" - redis: Final = _SharedRedis() - _process(monkeypatch, redis, cached_spend=6.0) - read: Final = asyncio.Event() - seeded: Final = asyncio.Event() - - async def _stale_request() -> None: - with spend_counter_batch_scope(cast(RedisCache, redis), frozenset((counter,))): - await _read_cold(counter) - read.set() - await seeded.wait() - del redis.values[counter] - await ps.ensure_spend_counter_initialized(counter_key=counter, source_cache_key=SOURCE) - - async def _first_request() -> None: - await read.wait() - await _cold_start(redis, counter) - seeded.set() - - await asyncio.gather(_stale_request(), _first_request()) - - assert redis.values[counter] == 6.0 - - -@pytest.mark.asyncio -async def test_another_process_seeding_after_this_request_read_the_counter_still_sums( - monkeypatch: pytest.MonkeyPatch, -) -> None: - counter: Final = "spend:end_user:two-processes" - redis: Final = _SharedRedis() - read: Final = asyncio.Event() - a_seeded: Final = asyncio.Event() - - async def _process_b() -> None: - with spend_counter_batch_scope(cast(RedisCache, redis), frozenset((counter,))): - await _read_cold(counter) - read.set() - await a_seeded.wait() - _process(monkeypatch, redis, cached_spend=5.0) - await ps.ensure_spend_counter_initialized(counter_key=counter, source_cache_key=SOURCE) - - async def _process_a() -> None: - await read.wait() - _process(monkeypatch, redis, cached_spend=3.0) - await _cold_start(redis, counter) - a_seeded.set() - - await asyncio.gather(_process_b(), _process_a()) - - assert redis.values[counter] == 8.0 - - -@pytest.mark.asyncio -async def test_a_failed_seed_fails_the_requests_that_joined_it_and_the_next_cold_start_seeds( - monkeypatch: pytest.MonkeyPatch, -) -> None: - counter: Final = "spend:end_user:failed-seed" - redis: Final = _SharedRedis() - _process(monkeypatch, redis, cached_spend=6.0) - redis.failing = True - - outcomes: Final = await asyncio.gather(*(_cold_start(redis, counter) for _ in range(3)), return_exceptions=True) - redis.failing = False - await _cold_start(redis, counter) - - assert [type(outcome) for outcome in outcomes] == [ConnectionError] * 3 - assert redis.values[counter] == 6.0 - - -class _HeldReplyRedis(_SharedRedis): - """Applies an increment at once but holds its reply until released, like a slow network round trip.""" - - def __init__(self) -> None: - super().__init__() - self.applied: Final = asyncio.Event() - self.release: Final = asyncio.Event() - - async def async_increment(self, key: str, value: float, **_: object) -> float: - self.values[key] = self.values.get(key, 0.0) + value - total: Final = self.values[key] - self.applied.set() - await self.release.wait() - return total - - -@pytest.mark.asyncio -async def test_a_counter_deleted_before_an_earlier_add_returns_is_seeded_by_a_later_cold_reader( - monkeypatch: pytest.MonkeyPatch, -) -> None: - counter: Final = "spend:end_user:deleted-mid-add" - redis: Final = _HeldReplyRedis() - _process(monkeypatch, redis, cached_spend=6.0) - - first: Final = asyncio.ensure_future(_cold_start(redis, counter)) - await redis.applied.wait() - del redis.values[counter] - later: Final = asyncio.ensure_future(_cold_start(redis, counter)) - for _ in range(20): - await asyncio.sleep(0) - redis.release.set() - await asyncio.gather(first, later) - - assert redis.values[counter] == 6.0 - - -def test_a_cold_start_on_another_event_loop_adds_on_its_own(monkeypatch: pytest.MonkeyPatch) -> None: - counter: Final = "spend:end_user:two-loops" - held: Final = threading.Event() - release: Final = threading.Event() - - class _ThreadHeldRedis(_SharedRedis): - async def async_increment(self, key: str, value: float, **_: object) -> float: - if threading.current_thread() is not threading.main_thread(): - held.set() - while not release.is_set(): - await asyncio.sleep(0.001) - return await super().async_increment(key, value) - - redis: Final = _ThreadHeldRedis() - _process(monkeypatch, redis, cached_spend=6.0) - other_loop: Final = threading.Thread(target=lambda: asyncio.run(_cold_start(redis, counter)), daemon=True) - other_loop.start() - assert held.wait(5) - - try: - asyncio.run(_cold_start(redis, counter)) - finally: - release.set() - other_loop.join(5) - - assert redis.values[counter] == 12.0 diff --git a/tests/unit/proxy/proxy_server/test_spend_counters.py b/tests/unit/proxy/proxy_server/test_spend_counters.py index d273ef8f42f..ac5e3ff2388 100644 --- a/tests/unit/proxy/proxy_server/test_spend_counters.py +++ b/tests/unit/proxy/proxy_server/test_spend_counters.py @@ -22,6 +22,7 @@ Pins covered: from __future__ import annotations import asyncio +import threading from collections.abc import Awaitable, Callable from datetime import datetime from typing import Final, Literal, cast @@ -36,8 +37,9 @@ from opentelemetry.trace import StatusCode import litellm import litellm.proxy.proxy_server as ps -from litellm._service_logger import ServiceLogging from litellm._internal_context import current_service_target +from litellm._service_logger import ServiceLogging +from litellm.caching.dual_cache import DualCache from litellm.caching.in_memory_cache import InMemoryCache from litellm.caching.redis_cache import RedisCache, RedisCircuitBreaker from litellm.constants import GLOBAL_PROXY_SPEND_CACHE_KEY @@ -50,6 +52,7 @@ from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import evict_loca from litellm.proxy.common_utils.cache_pydantic_utils import CacheCodec from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache, end_user_cache_key, tag_cache_key from litellm.proxy.management_endpoints.team_endpoints import refresh_cached_team +from litellm.proxy.spend_tracking.spend_counter_batch import active_spend_counter_batch, spend_counter_batch_scope from .conftest import normalize @@ -1817,7 +1820,9 @@ class _RedisCacheOnClient(RedisCache): class _FakeRedisHoldingOneReply(FakeAsyncRedis): - def __init__(self, server: FakeServer, held_increment: float, applied: asyncio.Event, released: asyncio.Event) -> None: + def __init__( + self, server: FakeServer, held_increment: float, applied: asyncio.Event, released: asyncio.Event + ) -> None: super().__init__(server=server) self._held_increment: Final = held_increment self._applied: Final = applied @@ -1970,9 +1975,7 @@ async def test_update_cache_traces_the_global_spend_redis_call_under_the_request for span in exporter.get_finished_spans() if span.name == "redis" and span.attributes is not None and span.parent is not None ) - assert redis_spans == ( - ("async_increment_if_present", expected_status, request_span.get_span_context().span_id), - ) + assert redis_spans == (("async_increment_if_present", expected_status, request_span.get_span_context().span_id),) class _UnwritableKeyUserApiKeyCache(UserApiKeyCache): @@ -2013,3 +2016,237 @@ async def test_update_cache_still_adds_spend_to_the_others_when_one_cached_objec 1.25, 10.25, ) + + +# Overlapping cold starts of one spend counter inside one process add that process's cached spend once. + +_COLD_SEED_SOURCE: Final = "end_user_id:customer-1" + + +class _SharedRedis: + """The counter commands a cold start sends, yielding like a network call so overlapping requests interleave.""" + + def __init__(self) -> None: + self.values: Final[dict[str, float]] = {} # mutable-ok: stands in for the Redis keyspace + self.failing = False + + async def async_batch_get_cache(self, key_list: list[str], **_: object) -> dict[str, float | None]: + await asyncio.sleep(0) + return {key: self.values.get(key) for key in key_list} + + async def async_get_cache(self, key: str, **_: object) -> float | None: + await asyncio.sleep(0) + return self.values.get(key) + + async def async_increment(self, key: str, value: float, **_: object) -> float: + await asyncio.sleep(0) + if self.failing: + raise ConnectionError("redis down") + self.values[key] = self.values.get(key, 0.0) + value + return self.values[key] + + async def async_delete_cache(self, key: str) -> None: + await asyncio.sleep(0) + if not self.failing: + self.values.pop(key, None) + + +def _process(monkeypatch: pytest.MonkeyPatch, redis: _SharedRedis | None, cached_spend: float) -> DualCache: + counters: Final = DualCache(in_memory_cache=InMemoryCache(), redis_cache=cast(RedisCache, redis)) + objects: Final = DualCache(in_memory_cache=InMemoryCache()) + objects.in_memory_cache.set_cache(key=_COLD_SEED_SOURCE, value={"spend": cached_spend}) + monkeypatch.setattr(ps, "spend_counter_cache", counters) + monkeypatch.setattr(ps, "user_api_key_cache", objects) + monkeypatch.setattr(ps, "prisma_client", object()) + return counters + + +async def _cold_start(redis: _SharedRedis | None, counter: str) -> None: + with spend_counter_batch_scope(cast(RedisCache, redis) if redis is not None else None, frozenset((counter,))): + await ps.ensure_spend_counter_initialized(counter_key=counter, source_cache_key=_COLD_SEED_SOURCE) + + +async def _read_cold(counter: str) -> None: + batch: Final = active_spend_counter_batch() + assert batch is not None + assert await batch.read(counter) == (None, True) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("with_redis", (True, False)) +async def test_overlapping_cold_starts_in_one_process_add_its_cached_spend_once( + monkeypatch: pytest.MonkeyPatch, with_redis: bool +) -> None: + counter: Final = f"spend:end_user:once-{with_redis}" + redis: Final = _SharedRedis() if with_redis else None + counters: Final = _process(monkeypatch, redis, cached_spend=6.0) + + await asyncio.gather(*(_cold_start(redis, counter) for _ in range(5))) + + seeded: Final = redis.values.get(counter) if redis is not None else counters.in_memory_cache.get_cache(key=counter) + assert seeded == 6.0 + + +@pytest.mark.asyncio +async def test_a_request_that_read_the_counter_before_this_process_seeded_it_does_not_add_again( + monkeypatch: pytest.MonkeyPatch, +) -> None: + counter: Final = "spend:end_user:stale-reader" + redis: Final = _SharedRedis() + _process(monkeypatch, redis, cached_spend=6.0) + read: Final = asyncio.Event() + + async def _stale_request() -> None: + with spend_counter_batch_scope(cast(RedisCache, redis), frozenset((counter,))): + await _read_cold(counter) + read.set() + await seeded.wait() + await ps.ensure_spend_counter_initialized(counter_key=counter, source_cache_key=_COLD_SEED_SOURCE) + + seeded: Final = asyncio.Event() + + async def _first_request() -> None: + await read.wait() + await _cold_start(redis, counter) + seeded.set() + + await asyncio.gather(_stale_request(), _first_request()) + + assert redis.values[counter] == 6.0 + + +@pytest.mark.asyncio +async def test_a_counter_lost_again_after_this_process_seeded_it_is_seeded_again( + monkeypatch: pytest.MonkeyPatch, +) -> None: + counter: Final = "spend:end_user:lost-again" + redis: Final = _SharedRedis() + _process(monkeypatch, redis, cached_spend=6.0) + read: Final = asyncio.Event() + seeded: Final = asyncio.Event() + + async def _stale_request() -> None: + with spend_counter_batch_scope(cast(RedisCache, redis), frozenset((counter,))): + await _read_cold(counter) + read.set() + await seeded.wait() + del redis.values[counter] + await ps.ensure_spend_counter_initialized(counter_key=counter, source_cache_key=_COLD_SEED_SOURCE) + + async def _first_request() -> None: + await read.wait() + await _cold_start(redis, counter) + seeded.set() + + await asyncio.gather(_stale_request(), _first_request()) + + assert redis.values[counter] == 6.0 + + +@pytest.mark.asyncio +async def test_another_process_seeding_after_this_request_read_the_counter_still_sums( + monkeypatch: pytest.MonkeyPatch, +) -> None: + counter: Final = "spend:end_user:two-processes" + redis: Final = _SharedRedis() + read: Final = asyncio.Event() + a_seeded: Final = asyncio.Event() + + async def _process_b() -> None: + with spend_counter_batch_scope(cast(RedisCache, redis), frozenset((counter,))): + await _read_cold(counter) + read.set() + await a_seeded.wait() + _process(monkeypatch, redis, cached_spend=5.0) + await ps.ensure_spend_counter_initialized(counter_key=counter, source_cache_key=_COLD_SEED_SOURCE) + + async def _process_a() -> None: + await read.wait() + _process(monkeypatch, redis, cached_spend=3.0) + await _cold_start(redis, counter) + a_seeded.set() + + await asyncio.gather(_process_b(), _process_a()) + + assert redis.values[counter] == 8.0 + + +@pytest.mark.asyncio +async def test_a_failed_seed_fails_the_requests_that_joined_it_and_the_next_cold_start_seeds( + monkeypatch: pytest.MonkeyPatch, +) -> None: + counter: Final = "spend:end_user:failed-seed" + redis: Final = _SharedRedis() + _process(monkeypatch, redis, cached_spend=6.0) + redis.failing = True + + outcomes: Final = await asyncio.gather(*(_cold_start(redis, counter) for _ in range(3)), return_exceptions=True) + redis.failing = False + await _cold_start(redis, counter) + + assert [type(outcome) for outcome in outcomes] == [ConnectionError] * 3 + assert redis.values[counter] == 6.0 + + +class _HeldReplyRedis(_SharedRedis): + """Applies an increment at once but holds its reply until released, like a slow network round trip.""" + + def __init__(self) -> None: + super().__init__() + self.applied: Final = asyncio.Event() + self.release: Final = asyncio.Event() + + async def async_increment(self, key: str, value: float, **_: object) -> float: + self.values[key] = self.values.get(key, 0.0) + value + total: Final = self.values[key] + self.applied.set() + await self.release.wait() + return total + + +@pytest.mark.asyncio +async def test_a_counter_deleted_before_an_earlier_add_returns_is_seeded_by_a_later_cold_reader( + monkeypatch: pytest.MonkeyPatch, +) -> None: + counter: Final = "spend:end_user:deleted-mid-add" + redis: Final = _HeldReplyRedis() + _process(monkeypatch, redis, cached_spend=6.0) + + first: Final = asyncio.ensure_future(_cold_start(redis, counter)) + await redis.applied.wait() + del redis.values[counter] + later: Final = asyncio.ensure_future(_cold_start(redis, counter)) + for _ in range(20): + await asyncio.sleep(0) + redis.release.set() + await asyncio.gather(first, later) + + assert redis.values[counter] == 6.0 + + +def test_a_cold_start_on_another_event_loop_adds_on_its_own(monkeypatch: pytest.MonkeyPatch) -> None: + counter: Final = "spend:end_user:two-loops" + held: Final = threading.Event() + release: Final = threading.Event() + + class _ThreadHeldRedis(_SharedRedis): + async def async_increment(self, key: str, value: float, **_: object) -> float: + if threading.current_thread() is not threading.main_thread(): + held.set() + while not release.is_set(): + await asyncio.sleep(0.001) + return await super().async_increment(key, value) + + redis: Final = _ThreadHeldRedis() + _process(monkeypatch, redis, cached_spend=6.0) + other_loop: Final = threading.Thread(target=lambda: asyncio.run(_cold_start(redis, counter)), daemon=True) + other_loop.start() + assert held.wait(5) + + try: + asyncio.run(_cold_start(redis, counter)) + finally: + release.set() + other_loop.join(5) + + assert redis.values[counter] == 12.0 From d2122a36bc8e6c9cba748e278bc48730e54bbc0b Mon Sep 17 00:00:00 2001 From: yucheng Date: Sat, 31 Oct 2026 15:11:58 -0700 Subject: [PATCH 5/7] fix(proxy): retry a failed shared cold seed, stamp the real read tick, and skip sharing when the Redis breaker is not closed A request that joined another request's cold seed starts or joins a new seed when that one fails, so one dropped connection fails only the request that owned it. A request waits on other requests' seeds for at most 50 ms and then adds its own. A request that finds the Redis circuit breaker not closed adds on its own, so it fails fast as on main. The batch now stamps each counter read with the tick its MGET returned, so a request that read the counter cold after this process's seed landed still adds its cached spend --- litellm/caching/redis_cache.py | 5 + litellm/constants.py | 1 + litellm/proxy/db/spend_counter_reseed.py | 51 +++-- litellm/proxy/proxy_server.py | 2 +- .../spend_tracking/spend_counter_batch.py | 42 +++-- .../proxy/proxy_server/test_spend_counters.py | 174 +++++++++++++++++- 6 files changed, 239 insertions(+), 36 deletions(-) diff --git a/litellm/caching/redis_cache.py b/litellm/caching/redis_cache.py index afe6f8a1eab..3b90cce211d 100644 --- a/litellm/caching/redis_cache.py +++ b/litellm/caching/redis_cache.py @@ -709,6 +709,11 @@ class RedisCache(BaseCache): def _admit_async_call(self, call: Callable[[], Awaitable[_RedisCallResult]]) -> Awaitable[_RedisCallResult]: return call() + @property + def circuit_breaker_closed(self) -> bool: + """Whether calls reach Redis without a recovery probe. Reading it, unlike making a call, never starts one.""" + return not self._circuit_breaker.enabled or self._circuit_breaker.state == RedisCircuitBreaker.CLOSED + def _setup_health_pings(self): """Setup async and sync health pings for Redis.""" # ASYNC HEALTH PING diff --git a/litellm/constants.py b/litellm/constants.py index 28e7b4a7fd7..199d7bc59bc 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1924,6 +1924,7 @@ SPEND_LOG_QUEUE_POLL_INTERVAL: Final = get_env_float("SPEND_LOG_QUEUE_POLL_INTER RESPONSES_SESSION_LOOKUP_MAX_ATTEMPTS: Final = max(1, get_env_int("RESPONSES_SESSION_LOOKUP_MAX_ATTEMPTS", 3)) RESPONSES_SESSION_LOOKUP_RETRY_INTERVAL: Final = get_env_float("RESPONSES_SESSION_LOOKUP_RETRY_INTERVAL", 0.2) SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE: Final = get_env_int("SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE", 10000) +SPEND_COUNTER_COLD_SEED_JOIN_SECONDS: Final = 0.05 PROXY_DB_LOOKUP_MAX_CONCURRENCY: Final = max(1, get_env_int("PROXY_DB_LOOKUP_MAX_CONCURRENCY", 25)) PROXY_DB_LOOKUP_DEADLINE_SECONDS: Final = max(0.1, get_env_float("PROXY_DB_LOOKUP_DEADLINE_SECONDS", 10.0)) PROXY_DB_LOOKUP_STALL_WINDOW_SECONDS: Final = max(0.0, get_env_float("PROXY_DB_LOOKUP_STALL_WINDOW_SECONDS", 30.0)) diff --git a/litellm/proxy/db/spend_counter_reseed.py b/litellm/proxy/db/spend_counter_reseed.py index 1d9eb5f95b2..2da5f0128e1 100644 --- a/litellm/proxy/db/spend_counter_reseed.py +++ b/litellm/proxy/db/spend_counter_reseed.py @@ -23,7 +23,7 @@ from typing import TYPE_CHECKING, ClassVar, Final, Optional from litellm._internal_context import with_service_target from litellm._logging import verbose_proxy_logger -from litellm.constants import SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE +from litellm.constants import SPEND_COUNTER_COLD_SEED_JOIN_SECONDS, SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE from litellm.litellm_core_utils.duration_parser import duration_in_seconds from litellm.proxy._types import Litellm_EntityType from litellm.proxy.db.db_lookup_gate import bounded_db_lookup, db_lookup_gate @@ -92,7 +92,8 @@ class _CachedSpendSeed: return in_flight def _settle(self, done: asyncio.Future[float | None]) -> None: - self.in_flight = None + if self.in_flight is done: + self.in_flight = None if not done.cancelled() and done.exception() is None: self.landed_at = spend_counter_tick() @@ -102,6 +103,11 @@ def _cached_spend_seed(spend_counter_cache: "DualCache", counter_key: str) -> _C return _CachedSpendSeed() +def _redis_breaker_closed(spend_counter_cache: "DualCache") -> bool: + redis_cache: Final = spend_counter_cache.redis_cache + return redis_cache is None or redis_cache.circuit_breaker_closed + + def _as_utc(value: datetime) -> datetime: return value if value.tzinfo is not None else value.replace(tzinfo=timezone.utc) @@ -155,20 +161,35 @@ class SpendCounterReseed: is_warm: Callable[[], Awaitable[bool]], ) -> None: """Run ``add``, this process adding its cached spend to a cold counter, once for all of this process's - requests that read the counter cold (at or after tick ``read_at``) before that add landed. A request joins an - add that started after its read and gets its result or error. When an add landed after its read, or one that - started before its read finishes, the request re-reads the counter and adds again only if the counter is gone. - A request on another event loop than the add in flight, and each other process, still adds its own cached - spend.""" + requests that read the counter cold (at tick ``read_at``) before that add landed. A request joins an add that + started after its read and gets its result, and starts or joins another add when that one fails. When an add + landed after its read, or one that started before its read finishes, the request re-reads the counter and adds + again only if the counter is gone. A request waits on other requests' adds for at most + ``SPEND_COUNTER_COLD_SEED_JOIN_SECONDS`` in all and then adds its own. A request on another event loop than + the add in flight, one that finds the Redis circuit breaker not closed, and each other process, still add their + own cached spend.""" seed: Final = _cached_spend_seed(spend_counter_cache, counter_key) - earlier: Final = seed.in_flight - if earlier is not None and seed.started_at < read_at and seed.on_this_loop(): - await asyncio.wait((earlier,)) - if seed.in_flight is None and seed.landed_at > read_at: - forget_spend_counter(counter_key) - if await is_warm(): - return - added: Final = await (asyncio.shield(seed.in_flight or seed.start(add)) if seed.on_this_loop() else add()) + loop: Final = asyncio.get_running_loop() + join_by: Final = loop.time() + SPEND_COUNTER_COLD_SEED_JOIN_SECONDS + + async def add_unless_covered(cold_at: int) -> float | None: + in_flight: Final = seed.in_flight + if not seed.on_this_loop() or not _redis_breaker_closed(spend_counter_cache): + return await add() + if in_flight is None: + if seed.landed_at <= cold_at: + return await asyncio.shield(seed.start(add)) + forget_spend_counter(counter_key) + return None if await is_warm() else await add_unless_covered(spend_counter_tick()) + covers: Final = seed.started_at > cold_at + await asyncio.wait((in_flight,), timeout=max(0.0, join_by - loop.time())) + if not in_flight.done(): + return await asyncio.shield(seed.start(add)) + if covers and not in_flight.cancelled() and in_flight.exception() is None: + return in_flight.result() + return await add_unless_covered(cold_at) + + added: Final = await add_unless_covered(read_at) if added is not None: record_spend_counter_value(counter_key, added) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index b43733286cd..c29ff0bee4a 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -4086,8 +4086,8 @@ async def ensure_spend_counter_initialized( counter_key: str, source_cache_key: str | list[str], ) -> None: - read_at: Final = spend_counter_read_tick() is_warm: Final = await _is_spend_counter_cache_warm(counter_key=counter_key) + read_at: Final = spend_counter_read_tick(counter_key) if is_warm is False: # Shares the per-counter lock with get_current_spend. db_spend: Final = await SpendCounterReseed.coalesced( diff --git a/litellm/proxy/spend_tracking/spend_counter_batch.py b/litellm/proxy/spend_tracking/spend_counter_batch.py index 49280448f75..4937fea039b 100644 --- a/litellm/proxy/spend_tracking/spend_counter_batch.py +++ b/litellm/proxy/spend_tracking/spend_counter_batch.py @@ -6,7 +6,7 @@ from collections.abc import Iterator, Mapping, Sequence from contextvars import ContextVar, Token from dataclasses import dataclass from types import MappingProxyType, TracebackType -from typing import Final +from typing import Final, TypeAlias from pydantic import TypeAdapter @@ -22,6 +22,8 @@ from litellm.proxy.common_utils.user_api_key_cache import ( _CounterValues: Final = TypeAdapter(dict[str, float | None]) _NO_VALUES: Final[Mapping[str, float | None]] = MappingProxyType({}) +_NO_TICKS: Final[Mapping[str, int]] = MappingProxyType({}) +_PendingMGet: TypeAlias = tuple[BatchResult[Mapping[str, object]], "asyncio.Future[int]"] SPEND_COUNTERS_TARGET: Final = "spend_counters" _ticks: Final = itertools.count(1) @@ -53,20 +55,20 @@ class SpendCounterBatch: "_loaded", "_lock", "_open", + "_read_at", "_redis_cache", "_request_batch", - "opened_at", ) def __init__(self, redis_cache: RedisCache) -> None: self._redis_cache: Final = redis_cache - self.opened_at: Final = spend_counter_tick() self._lock: Final = asyncio.Lock() self._open = True self._keys: frozenset[str] = frozenset() self._fetched: frozenset[str] = frozenset() self._loaded: Mapping[str, float | None] = _NO_VALUES - self._inflight: Final[list[BatchResult[Mapping[str, object]]]] = [] # mutable-ok: drained by _load + self._read_at: Mapping[str, int] = _NO_TICKS + self._inflight: Final[list[_PendingMGet]] = [] # mutable-ok: drained by _load self._request_batch: Final[RedisBatch | None] = active_request_redis_batch(redis_cache) if self._request_batch is not None: self._request_batch.add_flush_hook(self._declare_pending) @@ -96,6 +98,10 @@ class SpendCounterBatch: return None return loaded[counter_key], True + def read_tick(self, counter_key: str) -> int | None: + """The tick at which the MGET behind this batch's value for the counter returned, None without that value.""" + return self._read_at.get(counter_key) if self._open and counter_key in self._loaded else None + def record(self, counter_key: str, value: float) -> None: """A write returned the counter's new value; later reads in this scope see it instead of the MGET value.""" if not self._open: @@ -111,6 +117,7 @@ class SpendCounterBatch: self._keys = self._keys - key self._fetched = self._fetched - key self._loaded = MappingProxyType({k: v for k, v in self._loaded.items() if k != counter_key}) + self._read_at = MappingProxyType({k: v for k, v in self._read_at.items() if k != counter_key}) async def _load(self) -> Mapping[str, float | None]: async with self._lock: @@ -121,10 +128,13 @@ class SpendCounterBatch: pending: Final = self._keys - self._fetched if pending: self._fetched = self._fetched | pending - fetched: Final = await self._fetch(pending) - self._loaded = MappingProxyType({**fetched, **self._loaded}) + self._merge(await self._fetch(pending), spend_counter_tick()) return self._loaded + def _merge(self, fetched: Mapping[str, float | None], read_at: int) -> None: + self._loaded = MappingProxyType({**fetched, **self._loaded}) + self._read_at = MappingProxyType({**dict.fromkeys(fetched, read_at), **self._read_at}) + def _declare_pending(self) -> None: """Flush hook: put every bound-but-unread counter on the request pipeline that is about to go out.""" if self._request_batch is None or not self._open: @@ -133,18 +143,21 @@ class SpendCounterBatch: if pending: self._fetched = self._fetched | pending with service_target(SPEND_COUNTERS_TARGET): - self._inflight.append(self._request_batch.mget(sorted(pending))) + read: Final = self._request_batch.mget(sorted(pending)) + returned_at: Final[asyncio.Future[int]] = asyncio.get_running_loop().create_future() + read.on_settled(lambda _: returned_at.set_result(spend_counter_tick())) + self._inflight.append((read, returned_at)) async def _collect_inflight(self) -> None: - results: Final = tuple(self._inflight) + reads: Final = tuple(self._inflight) self._inflight.clear() - for result in results: + for read, returned_at in reads: try: - fetched: Mapping[str, float | None] = _CounterValues.validate_python(await result) + fetched: Mapping[str, float | None] = _CounterValues.validate_python(await read) except Exception as e: # noqa: BLE001 # per-key reads take over and apply their own Redis fallback verbose_proxy_logger.debug("spend counter batch read failed, falling back to per-key reads: %s", e) continue - self._loaded = MappingProxyType({**fetched, **self._loaded}) + self._merge(fetched, returned_at.result() if returned_at.done() else spend_counter_tick()) async def _fetch(self, keys: frozenset[str]) -> Mapping[str, float | None]: try: @@ -163,10 +176,11 @@ def active_spend_counter_batch() -> SpendCounterBatch | None: return _active_batch.get() -def spend_counter_read_tick() -> int: - """A tick no later than this context's next spend counter read: the open batch's opening tick, else a new one.""" +def spend_counter_read_tick(counter_key: str) -> int: + """The tick this context last read the counter at: when the open batch's MGET for it returned, else now.""" batch: Final = _active_batch.get() - return batch.opened_at if batch is not None and batch.is_open else spend_counter_tick() + read_at: Final = batch.read_tick(counter_key) if batch is not None else None + return read_at if read_at is not None else spend_counter_tick() class spend_counter_batch_scope: diff --git a/tests/unit/proxy/proxy_server/test_spend_counters.py b/tests/unit/proxy/proxy_server/test_spend_counters.py index ac5e3ff2388..06c8444f2d8 100644 --- a/tests/unit/proxy/proxy_server/test_spend_counters.py +++ b/tests/unit/proxy/proxy_server/test_spend_counters.py @@ -41,7 +41,8 @@ from litellm._internal_context import current_service_target from litellm._service_logger import ServiceLogging from litellm.caching.dual_cache import DualCache from litellm.caching.in_memory_cache import InMemoryCache -from litellm.caching.redis_cache import RedisCache, RedisCircuitBreaker +from litellm.caching.redis_batch import request_redis_batch_scope +from litellm.caching.redis_cache import RedisCache, RedisCircuitBreaker, RedisCircuitBreakerOpenError from litellm.constants import GLOBAL_PROXY_SPEND_CACHE_KEY from litellm.integrations.opentelemetry import OpenTelemetry, OpenTelemetryConfig from litellm.models.end_user import LiteLLM_EndUserTable @@ -1808,10 +1809,10 @@ class _FakeRedisRunningIncrementIfPresent(FakeAsyncRedis): class _RedisCacheOnClient(RedisCache): - def __init__(self, client: FakeAsyncRedis) -> None: + def __init__(self, client: FakeAsyncRedis, breaker: RedisCircuitBreaker | None = None) -> None: self._injected_client: Final = client self.namespace = None - self._circuit_breaker = RedisCircuitBreaker(failure_threshold=5, recovery_timeout=30) + self._circuit_breaker = breaker or RedisCircuitBreaker(failure_threshold=5, recovery_timeout=30) self.service_logger_obj = ServiceLogging() self.default_ttl = None @@ -2026,6 +2027,8 @@ _COLD_SEED_SOURCE: Final = "end_user_id:customer-1" class _SharedRedis: """The counter commands a cold start sends, yielding like a network call so overlapping requests interleave.""" + circuit_breaker_closed: Final = True + def __init__(self) -> None: self.values: Final[dict[str, float]] = {} # mutable-ok: stands in for the Redis keyspace self.failing = False @@ -2051,7 +2054,9 @@ class _SharedRedis: self.values.pop(key, None) -def _process(monkeypatch: pytest.MonkeyPatch, redis: _SharedRedis | None, cached_spend: float) -> DualCache: +def _process( + monkeypatch: pytest.MonkeyPatch, redis: _SharedRedis | RedisCache | None, cached_spend: float +) -> DualCache: counters: Final = DualCache(in_memory_cache=InMemoryCache(), redis_cache=cast(RedisCache, redis)) objects: Final = DualCache(in_memory_cache=InMemoryCache()) objects.in_memory_cache.set_cache(key=_COLD_SEED_SOURCE, value={"spend": cached_spend}) @@ -2061,7 +2066,7 @@ def _process(monkeypatch: pytest.MonkeyPatch, redis: _SharedRedis | None, cached return counters -async def _cold_start(redis: _SharedRedis | None, counter: str) -> None: +async def _cold_start(redis: _SharedRedis | RedisCache | None, counter: str) -> None: with spend_counter_batch_scope(cast(RedisCache, redis) if redis is not None else None, frozenset((counter,))): await ps.ensure_spend_counter_initialized(counter_key=counter, source_cache_key=_COLD_SEED_SOURCE) @@ -2172,7 +2177,7 @@ async def test_another_process_seeding_after_this_request_read_the_counter_still @pytest.mark.asyncio -async def test_a_failed_seed_fails_the_requests_that_joined_it_and_the_next_cold_start_seeds( +async def test_seeds_that_keep_failing_fail_every_cold_start_and_the_next_cold_start_seeds( monkeypatch: pytest.MonkeyPatch, ) -> None: counter: Final = "spend:end_user:failed-seed" @@ -2250,3 +2255,160 @@ def test_a_cold_start_on_another_event_loop_adds_on_its_own(monkeypatch: pytest. other_loop.join(5) assert redis.values[counter] == 12.0 + + +@pytest.mark.asyncio +async def test_a_request_that_read_the_counter_cold_after_this_process_seeded_it_still_adds_its_cached_spend( + monkeypatch: pytest.MonkeyPatch, +) -> None: + counter: Final = "spend:end_user:read-after-seed" + redis: Final = _SharedRedis() + _process(monkeypatch, redis, cached_spend=6.0) + opened: Final = asyncio.Event() + seeded: Final = asyncio.Event() + + async def _late_reader() -> None: + with spend_counter_batch_scope(cast(RedisCache, redis), frozenset((counter,))): + opened.set() + await seeded.wait() + del redis.values[counter] + await _read_cold(counter) + redis.values[counter] = 0.5 + await ps.ensure_spend_counter_initialized(counter_key=counter, source_cache_key=_COLD_SEED_SOURCE) + + async def _first_request() -> None: + await opened.wait() + await _cold_start(redis, counter) + seeded.set() + + await asyncio.gather(_late_reader(), _first_request()) + + assert redis.values[counter] == 6.5 + + +@pytest.mark.asyncio +async def test_a_request_whose_pipelined_read_returned_before_this_process_seeded_the_counter_does_not_add_again( + monkeypatch: pytest.MonkeyPatch, +) -> None: + counter: Final = "spend:end_user:pipelined-reader" + client: Final = FakeAsyncRedis(server=FakeServer()) + redis: Final = _RedisCacheOnClient(client) + _process(monkeypatch, redis, cached_spend=6.0) + read: Final = asyncio.Event() + seeded: Final = asyncio.Event() + + async def _pipelined_request() -> None: + with request_redis_batch_scope() as batches, spend_counter_batch_scope(redis, frozenset((counter,))): + await batches.batch(redis).flush() + read.set() + await seeded.wait() + await ps.ensure_spend_counter_initialized(counter_key=counter, source_cache_key=_COLD_SEED_SOURCE) + + async def _first_request() -> None: + await read.wait() + await _cold_start(redis, counter) + seeded.set() + + await asyncio.gather(_pipelined_request(), _first_request()) + + assert float(await client.get(counter) or 0) == 6.0 + + +class _DroppedFirstIncrementRedis(_SharedRedis): + """Resets the connection that carries the first increment and answers every later one.""" + + def __init__(self) -> None: + super().__init__() + self.dropped = False + + async def async_increment(self, key: str, value: float, **_: object) -> float: + if not self.dropped: + self.dropped = True + await asyncio.sleep(0) + raise ConnectionError("connection reset by peer") + return await super().async_increment(key, value) + + +@pytest.mark.asyncio +async def test_requests_that_joined_a_seed_whose_connection_reset_seed_again_and_only_its_starter_fails( + monkeypatch: pytest.MonkeyPatch, +) -> None: + counter: Final = "spend:end_user:reset-seed" + redis: Final = _DroppedFirstIncrementRedis() + _process(monkeypatch, redis, cached_spend=6.0) + + outcomes: Final = await asyncio.gather(*(_cold_start(redis, counter) for _ in range(4)), return_exceptions=True) + + failures: Final = [outcome for outcome in outcomes if outcome is not None] + assert [type(failure) for failure in failures] == [ConnectionError], outcomes + assert redis.values[counter] == 6.0 + + +class _StalledFirstIncrementRedis(_SharedRedis): + """Never answers the first increment until released, like a connection that stopped responding.""" + + def __init__(self) -> None: + super().__init__() + self.stalled: Final = asyncio.Event() + self.release: Final = asyncio.Event() + + async def async_increment(self, key: str, value: float, **_: object) -> float: + if not self.stalled.is_set(): + self.stalled.set() + await self.release.wait() + raise TimeoutError("redis did not answer") + return await super().async_increment(key, value) + + +@pytest.mark.asyncio +async def test_cold_starts_stop_waiting_on_a_stalled_seed_and_add_their_own(monkeypatch: pytest.MonkeyPatch) -> None: + counter: Final = "spend:end_user:stalled-seed" + redis: Final = _StalledFirstIncrementRedis() + _process(monkeypatch, redis, cached_spend=6.0) + stalled: Final = asyncio.ensure_future(_cold_start(redis, counter)) + await redis.stalled.wait() + + try: + await asyncio.wait_for(asyncio.gather(*(_cold_start(redis, counter) for _ in range(3))), timeout=10) + seeded_while_stalled: Final = redis.values.get(counter) + finally: + redis.release.set() + + with pytest.raises(TimeoutError): + await stalled + assert seeded_while_stalled == 18.0 + + +class _FakeRedisHoldingIncrements(FakeAsyncRedis): + def __init__(self, server: FakeServer) -> None: + super().__init__(server=server) + self.holding: Final = asyncio.Event() + self.release: Final = asyncio.Event() + + async def incrbyfloat(self, name: str, amount: float = 1.0) -> float: + self.holding.set() + await self.release.wait() + return await super().incrbyfloat(name, amount) + + +@pytest.mark.asyncio +async def test_a_cold_start_after_the_redis_breaker_opened_fails_fast_instead_of_using_an_add_in_flight( + monkeypatch: pytest.MonkeyPatch, +) -> None: + counter: Final = "spend:end_user:breaker-open" + breaker: Final = RedisCircuitBreaker(failure_threshold=1, recovery_timeout=30) + client: Final = _FakeRedisHoldingIncrements(FakeServer()) + redis: Final = _RedisCacheOnClient(client, breaker) + _process(monkeypatch, redis, cached_spend=6.0) + first: Final = asyncio.ensure_future(_cold_start(redis, counter)) + await client.holding.wait() + breaker.record_failure() + + later: Final = asyncio.ensure_future(_cold_start(redis, counter)) + for _ in range(100): + await asyncio.sleep(0) + client.release.set() + + outcomes: Final = await asyncio.gather(first, later, return_exceptions=True) + + assert outcomes[0] is None and isinstance(outcomes[1], RedisCircuitBreakerOpenError), outcomes From 7bf40641a8ed72c25692299eb9ee89e0b753f9c9 Mon Sep 17 00:00:00 2001 From: yucheng Date: Sat, 31 Oct 2026 15:38:47 -0700 Subject: [PATCH 6/7] fix(proxy): compare cold seeds to when a read went out and came back A cold read now carries two ticks, when it was sent and when its reply came back A request adds its cached spend directly only when this process's last add landed before the read was sent, and otherwise re-reads the counter A re-read is stamped before its GET goes out, and a pipelined read keeps the tick from before its MGET, so a seed that lands while a read is in flight is never added twice A request joins an add in flight only when that add started after its read came back --- litellm/proxy/db/spend_counter_reseed.py | 26 +-- litellm/proxy/proxy_server.py | 8 +- .../spend_tracking/spend_counter_batch.py | 55 ++++-- .../proxy/proxy_server/test_spend_counters.py | 160 ++++++++++++++++++ 4 files changed, 217 insertions(+), 32 deletions(-) diff --git a/litellm/proxy/db/spend_counter_reseed.py b/litellm/proxy/db/spend_counter_reseed.py index 2da5f0128e1..4faca60779d 100644 --- a/litellm/proxy/db/spend_counter_reseed.py +++ b/litellm/proxy/db/spend_counter_reseed.py @@ -29,6 +29,7 @@ from litellm.proxy._types import Litellm_EntityType from litellm.proxy.db.db_lookup_gate import bounded_db_lookup, db_lookup_gate from litellm.proxy.spend_tracking.spend_counter_batch import ( SPEND_COUNTERS_TARGET, + SpendCounterRead, forget_spend_counter, read_batched_spend_counter, record_spend_counter_value, @@ -156,15 +157,15 @@ class SpendCounterReseed: async def add_cached_spend_once( spend_counter_cache: "DualCache", counter_key: str, - read_at: int, + read: SpendCounterRead, add: Callable[[], Awaitable[float | None]], is_warm: Callable[[], Awaitable[bool]], ) -> None: """Run ``add``, this process adding its cached spend to a cold counter, once for all of this process's - requests that read the counter cold (at tick ``read_at``) before that add landed. A request joins an add that - started after its read and gets its result, and starts or joins another add when that one fails. When an add - landed after its read, or one that started before its read finishes, the request re-reads the counter and adds - again only if the counter is gone. A request waits on other requests' adds for at most + requests that read the counter cold before that add landed. A request joins an add that started after its read + came back and gets its result, and starts or joins another add when that one fails. When an add landed after + its read went out, or one that started before its read came back finishes, the request re-reads the counter + and adds again only if the counter is gone. A request waits on other requests' adds for at most ``SPEND_COUNTER_COLD_SEED_JOIN_SECONDS`` in all and then adds its own. A request on another event loop than the add in flight, one that finds the Redis circuit breaker not closed, and each other process, still add their own cached spend.""" @@ -172,24 +173,27 @@ class SpendCounterReseed: loop: Final = asyncio.get_running_loop() join_by: Final = loop.time() + SPEND_COUNTER_COLD_SEED_JOIN_SECONDS - async def add_unless_covered(cold_at: int) -> float | None: + async def add_unless_covered(cold: SpendCounterRead) -> float | None: in_flight: Final = seed.in_flight if not seed.on_this_loop() or not _redis_breaker_closed(spend_counter_cache): return await add() if in_flight is None: - if seed.landed_at <= cold_at: + if seed.landed_at < cold.sent_at: return await asyncio.shield(seed.start(add)) forget_spend_counter(counter_key) - return None if await is_warm() else await add_unless_covered(spend_counter_tick()) - covers: Final = seed.started_at > cold_at + sent_at: Final = spend_counter_tick() + if await is_warm(): + return None + return await add_unless_covered(SpendCounterRead(sent_at=sent_at, returned_at=spend_counter_tick())) + covers: Final = seed.started_at > cold.returned_at await asyncio.wait((in_flight,), timeout=max(0.0, join_by - loop.time())) if not in_flight.done(): return await asyncio.shield(seed.start(add)) if covers and not in_flight.cancelled() and in_flight.exception() is None: return in_flight.result() - return await add_unless_covered(cold_at) + return await add_unless_covered(cold) - added: Final = await add_unless_covered(read_at) + added: Final = await add_unless_covered(read) if added is not None: record_spend_counter_value(counter_key, added) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index c29ff0bee4a..5a6e9756abb 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -936,7 +936,8 @@ from litellm.proxy.spend_tracking.spend_counter_batch import ( read_batched_spend_counter, record_spend_counter_value, spend_counter_batch_scope, - spend_counter_read_tick, + spend_counter_read, + spend_counter_tick, ) from litellm.proxy.spend_tracking.spend_management_endpoints import ( router as spend_management_router, @@ -4086,8 +4087,9 @@ async def ensure_spend_counter_initialized( counter_key: str, source_cache_key: str | list[str], ) -> None: + sent_at: Final = spend_counter_tick() is_warm: Final = await _is_spend_counter_cache_warm(counter_key=counter_key) - read_at: Final = spend_counter_read_tick(counter_key) + read: Final = spend_counter_read(counter_key, sent_at) if is_warm is False: # Shares the per-counter lock with get_current_spend. db_spend: Final = await SpendCounterReseed.coalesced( @@ -4103,7 +4105,7 @@ async def ensure_spend_counter_initialized( await SpendCounterReseed.add_cached_spend_once( spend_counter_cache=spend_counter_cache, counter_key=counter_key, - read_at=read_at, + read=read, add=lambda: increment_spend_counter_cache(counter_key=counter_key, increment=base_spend), is_warm=lambda: _is_spend_counter_cache_warm(counter_key=counter_key), ) diff --git a/litellm/proxy/spend_tracking/spend_counter_batch.py b/litellm/proxy/spend_tracking/spend_counter_batch.py index 4937fea039b..e15d3a62122 100644 --- a/litellm/proxy/spend_tracking/spend_counter_batch.py +++ b/litellm/proxy/spend_tracking/spend_counter_batch.py @@ -22,8 +22,6 @@ from litellm.proxy.common_utils.user_api_key_cache import ( _CounterValues: Final = TypeAdapter(dict[str, float | None]) _NO_VALUES: Final[Mapping[str, float | None]] = MappingProxyType({}) -_NO_TICKS: Final[Mapping[str, int]] = MappingProxyType({}) -_PendingMGet: TypeAlias = tuple[BatchResult[Mapping[str, object]], "asyncio.Future[int]"] SPEND_COUNTERS_TARGET: Final = "spend_counters" _ticks: Final = itertools.count(1) @@ -33,6 +31,18 @@ def spend_counter_tick() -> int: return next(_ticks) +@dataclass(frozen=True, slots=True) +class SpendCounterRead: + """When a read of a spend counter went out and when its reply came back, as spend counter ticks.""" + + sent_at: int + returned_at: int + + +_NO_READS: Final[Mapping[str, SpendCounterRead]] = MappingProxyType({}) +_PendingMGet: TypeAlias = tuple[BatchResult[Mapping[str, object]], int, "asyncio.Future[int]"] + + @dataclass(frozen=True, slots=True) class PendingSpendIncrement: counter_key: str @@ -55,7 +65,7 @@ class SpendCounterBatch: "_loaded", "_lock", "_open", - "_read_at", + "_reads", "_redis_cache", "_request_batch", ) @@ -67,7 +77,7 @@ class SpendCounterBatch: self._keys: frozenset[str] = frozenset() self._fetched: frozenset[str] = frozenset() self._loaded: Mapping[str, float | None] = _NO_VALUES - self._read_at: Mapping[str, int] = _NO_TICKS + self._reads: Mapping[str, SpendCounterRead] = _NO_READS self._inflight: Final[list[_PendingMGet]] = [] # mutable-ok: drained by _load self._request_batch: Final[RedisBatch | None] = active_request_redis_batch(redis_cache) if self._request_batch is not None: @@ -98,9 +108,9 @@ class SpendCounterBatch: return None return loaded[counter_key], True - def read_tick(self, counter_key: str) -> int | None: - """The tick at which the MGET behind this batch's value for the counter returned, None without that value.""" - return self._read_at.get(counter_key) if self._open and counter_key in self._loaded else None + def counter_read(self, counter_key: str) -> SpendCounterRead | None: + """When the MGET behind this batch's value for the counter went out and came back, None without that value.""" + return self._reads.get(counter_key) if self._open and counter_key in self._loaded else None def record(self, counter_key: str, value: float) -> None: """A write returned the counter's new value; later reads in this scope see it instead of the MGET value.""" @@ -117,7 +127,7 @@ class SpendCounterBatch: self._keys = self._keys - key self._fetched = self._fetched - key self._loaded = MappingProxyType({k: v for k, v in self._loaded.items() if k != counter_key}) - self._read_at = MappingProxyType({k: v for k, v in self._read_at.items() if k != counter_key}) + self._reads = MappingProxyType({k: v for k, v in self._reads.items() if k != counter_key}) async def _load(self) -> Mapping[str, float | None]: async with self._lock: @@ -128,12 +138,14 @@ class SpendCounterBatch: pending: Final = self._keys - self._fetched if pending: self._fetched = self._fetched | pending - self._merge(await self._fetch(pending), spend_counter_tick()) + sent_at: Final = spend_counter_tick() + fetched: Final = await self._fetch(pending) + self._merge(fetched, SpendCounterRead(sent_at=sent_at, returned_at=spend_counter_tick())) return self._loaded - def _merge(self, fetched: Mapping[str, float | None], read_at: int) -> None: + def _merge(self, fetched: Mapping[str, float | None], read: SpendCounterRead) -> None: self._loaded = MappingProxyType({**fetched, **self._loaded}) - self._read_at = MappingProxyType({**dict.fromkeys(fetched, read_at), **self._read_at}) + self._reads = MappingProxyType({**dict.fromkeys(fetched, read), **self._reads}) def _declare_pending(self) -> None: """Flush hook: put every bound-but-unread counter on the request pipeline that is about to go out.""" @@ -142,22 +154,28 @@ class SpendCounterBatch: pending: Final = self._keys - self._fetched if pending: self._fetched = self._fetched | pending + sent_at: Final = spend_counter_tick() with service_target(SPEND_COUNTERS_TARGET): read: Final = self._request_batch.mget(sorted(pending)) returned_at: Final[asyncio.Future[int]] = asyncio.get_running_loop().create_future() read.on_settled(lambda _: returned_at.set_result(spend_counter_tick())) - self._inflight.append((read, returned_at)) + self._inflight.append((read, sent_at, returned_at)) async def _collect_inflight(self) -> None: reads: Final = tuple(self._inflight) self._inflight.clear() - for read, returned_at in reads: + for read, sent_at, returned_at in reads: try: fetched: Mapping[str, float | None] = _CounterValues.validate_python(await read) except Exception as e: # noqa: BLE001 # per-key reads take over and apply their own Redis fallback verbose_proxy_logger.debug("spend counter batch read failed, falling back to per-key reads: %s", e) continue - self._merge(fetched, returned_at.result() if returned_at.done() else spend_counter_tick()) + self._merge( + fetched, + SpendCounterRead( + sent_at=sent_at, returned_at=returned_at.result() if returned_at.done() else spend_counter_tick() + ), + ) async def _fetch(self, keys: frozenset[str]) -> Mapping[str, float | None]: try: @@ -176,11 +194,12 @@ def active_spend_counter_batch() -> SpendCounterBatch | None: return _active_batch.get() -def spend_counter_read_tick(counter_key: str) -> int: - """The tick this context last read the counter at: when the open batch's MGET for it returned, else now.""" +def spend_counter_read(counter_key: str, sent_at: int) -> SpendCounterRead: + """When this context's last read of the counter went out and came back: the open batch's MGET for it, else a read + that went out at ``sent_at`` and came back now.""" batch: Final = _active_batch.get() - read_at: Final = batch.read_tick(counter_key) if batch is not None else None - return read_at if read_at is not None else spend_counter_tick() + batched: Final = batch.counter_read(counter_key) if batch is not None else None + return batched if batched is not None else SpendCounterRead(sent_at=sent_at, returned_at=spend_counter_tick()) class spend_counter_batch_scope: diff --git a/tests/unit/proxy/proxy_server/test_spend_counters.py b/tests/unit/proxy/proxy_server/test_spend_counters.py index 06c8444f2d8..f8e0dd16ea1 100644 --- a/tests/unit/proxy/proxy_server/test_spend_counters.py +++ b/tests/unit/proxy/proxy_server/test_spend_counters.py @@ -2412,3 +2412,163 @@ async def test_a_cold_start_after_the_redis_breaker_opened_fails_fast_instead_of outcomes: Final = await asyncio.gather(first, later, return_exceptions=True) assert outcomes[0] is None and isinstance(outcomes[1], RedisCircuitBreakerOpenError), outcomes + + +class _HeldFirstMGetRedis(_SharedRedis): + """Reads the first MGET's keys at once but holds its reply until released, like a slow network round trip.""" + + def __init__(self) -> None: + super().__init__() + self.mget_read: Final = asyncio.Event() + self.release_mget: Final = asyncio.Event() + + async def async_batch_get_cache(self, key_list: list[str], **_: object) -> dict[str, float | None]: + values: Final = {key: self.values.get(key) for key in key_list} + if not self.mget_read.is_set(): + self.mget_read.set() + await self.release_mget.wait() + return values + + +@pytest.mark.asyncio +async def test_a_seed_that_lands_while_a_cold_read_is_on_its_way_back_is_not_added_again( + monkeypatch: pytest.MonkeyPatch, +) -> None: + counter: Final = "spend:end_user:seed-during-read" + redis: Final = _HeldFirstMGetRedis() + _process(monkeypatch, redis, cached_spend=6.0) + + async def _first_request() -> None: + await redis.mget_read.wait() + await _cold_start(redis, counter) + redis.release_mget.set() + + await asyncio.gather(_cold_start(redis, counter), _first_request()) + + assert redis.values[counter] == 6.0 + + +class _HeldReadRedis(_SharedRedis): + """Answers an MGET with the values its keys had when it arrived, holding the reply while ``hold_reads`` is set.""" + + def __init__(self) -> None: + super().__init__() + self.hold_reads = False + self.read_arrived: Final = asyncio.Event() + self.release_read: Final = asyncio.Event() + + async def async_batch_get_cache(self, key_list: list[str], **_: object) -> dict[str, float | None]: + values: Final = {key: self.values.get(key) for key in key_list} + if self.hold_reads: + self.read_arrived.set() + await self.release_read.wait() + return values + + +@pytest.mark.asyncio +async def test_a_seed_that_lands_while_a_request_re_reads_the_counter_is_not_added_again( + monkeypatch: pytest.MonkeyPatch, +) -> None: + counter: Final = "spend:end_user:seed-during-reread" + redis: Final = _HeldReadRedis() + _process(monkeypatch, redis, cached_spend=6.0) + read: Final = asyncio.Event() + seeded: Final = asyncio.Event() + + async def _stale_request() -> None: + with spend_counter_batch_scope(cast(RedisCache, redis), frozenset((counter,))): + await _read_cold(counter) + read.set() + await seeded.wait() + del redis.values[counter] + redis.hold_reads = True + await ps.ensure_spend_counter_initialized(counter_key=counter, source_cache_key=_COLD_SEED_SOURCE) + + async def _first_request() -> None: + await read.wait() + await _cold_start(redis, counter) + seeded.set() + + async def _request_seeding_during_the_re_read() -> None: + await redis.read_arrived.wait() + redis.hold_reads = False + await _cold_start(redis, counter) + redis.release_read.set() + + await asyncio.gather(_stale_request(), _first_request(), _request_seeding_during_the_re_read()) + + assert redis.values[counter] == 6.0 + + +class _HeldGetRedis(_SharedRedis): + """Answers the first GET with the value its key had when it arrived, holding the reply until released.""" + + def __init__(self) -> None: + super().__init__() + self.get_arrived: Final = asyncio.Event() + self.release_get: Final = asyncio.Event() + + async def async_get_cache(self, key: str, **_: object) -> float | None: + value: Final = self.values.get(key) + if not self.get_arrived.is_set(): + self.get_arrived.set() + await self.release_get.wait() + return value + + +@pytest.mark.asyncio +async def test_a_seed_that_lands_while_an_unbatched_cold_read_is_on_its_way_back_is_not_added_again( + monkeypatch: pytest.MonkeyPatch, +) -> None: + counter: Final = "spend:end_user:seed-during-unbatched-read" + redis: Final = _HeldGetRedis() + _process(monkeypatch, redis, cached_spend=6.0) + + async def _first_request() -> None: + await redis.get_arrived.wait() + await _cold_start(redis, counter) + redis.release_get.set() + + await asyncio.gather( + ps.ensure_spend_counter_initialized(counter_key=counter, source_cache_key=_COLD_SEED_SOURCE), + _first_request(), + ) + + assert redis.values[counter] == 6.0 + + +class _QueuedReadRedis(_HeldReplyRedis): + """Holds the first MGET before Redis runs it, while increments apply at once and hold their replies.""" + + def __init__(self) -> None: + super().__init__() + self.read_sent: Final = asyncio.Event() + self.run_read: Final = asyncio.Event() + + async def async_batch_get_cache(self, key_list: list[str], **_: object) -> dict[str, float | None]: + if not self.read_sent.is_set(): + self.read_sent.set() + await self.run_read.wait() + return {key: self.values.get(key) for key in key_list} + + +@pytest.mark.asyncio +async def test_a_cold_read_sent_before_an_add_started_does_not_join_it_when_the_counter_was_deleted_after_it_landed( + monkeypatch: pytest.MonkeyPatch, +) -> None: + counter: Final = "spend:end_user:deleted-while-read-queued" + redis: Final = _QueuedReadRedis() + _process(monkeypatch, redis, cached_spend=6.0) + + reader: Final = asyncio.ensure_future(_cold_start(redis, counter)) + await redis.read_sent.wait() + first: Final = asyncio.ensure_future(_cold_start(redis, counter)) + await redis.applied.wait() + del redis.values[counter] + redis.run_read.set() + for _ in range(20): + await asyncio.sleep(0) + redis.release.set() + await asyncio.gather(reader, first) + + assert redis.values.get(counter) == 6.0 From 1944a38928a4ef5bfbc4fe498a5d0baf4922b430 Mon Sep 17 00:00:00 2001 From: yucheng Date: Sat, 31 Oct 2026 15:45:10 -0700 Subject: [PATCH 7/7] test(proxy): drop the banner comment above the cold seed tests --- tests/unit/proxy/proxy_server/test_spend_counters.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/unit/proxy/proxy_server/test_spend_counters.py b/tests/unit/proxy/proxy_server/test_spend_counters.py index f8e0dd16ea1..a361fe36051 100644 --- a/tests/unit/proxy/proxy_server/test_spend_counters.py +++ b/tests/unit/proxy/proxy_server/test_spend_counters.py @@ -2019,8 +2019,6 @@ async def test_update_cache_still_adds_spend_to_the_others_when_one_cached_objec ) -# Overlapping cold starts of one spend counter inside one process add that process's cached spend once. - _COLD_SEED_SOURCE: Final = "end_user_id:customer-1"