This commit is contained in:
yucheng-berri 2026-11-03 01:54:22 +00:00 • committed by GitHub
commit 26a049cfd1
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 1570 additions and 34 deletions

View file

@ -709,6 +709,11 @@ class RedisCache(BaseCache):
def _admit_async_call(self, call: Callable[[], Awaitable[_RedisCallResult]]) -> Awaitable[_RedisCallResult]:
return call()
@property
def circuit_breaker_closed(self) -> bool:
"""Whether calls reach Redis without a recovery probe. Reading it, unlike making a call, never starts one."""
return not self._circuit_breaker.enabled or self._circuit_breaker.state == RedisCircuitBreaker.CLOSED
def _setup_health_pings(self):
"""Setup async and sync health pings for Redis."""
# ASYNC HEALTH PING
@ -1468,15 +1473,29 @@ class RedisCache(BaseCache):
await _redis_client.expire(key, ttl)
return float(result)
async def _set_if_missing_with_ttl(self, key: str, value: float, ttl: int | None) -> float:
"""SET NX, its TTL command and a GET in one round trip; the value the key holds afterwards."""
async with self._async_commands().pipeline(transaction=False) as pipe:
pipe.set(name=key, value=value, nx=True)
if ttl is not None:
pipe.expire(key, ttl)
pipe.get(key)
replies: Final[Sequence[object]] = await pipe.execute() # pyright: ignore[reportUnknownVariableType, reportUnknownMemberType] # redis generics
current: Final = _LUA_OPTIONAL_TOTAL.validate_python(replies[-1])
return value if current is None else current
@_redis_circuit_breaker_guard
async def async_increment(
self,
key,
key: str,
value: float,
ttl: int | None = None,
parent_otel_span: Span | None = None,
refresh_ttl: bool = False,
only_if_missing: bool = False,
) -> float:
"""Add ``value`` to the key and return its new value. With ``only_if_missing``, a key that exists keeps its
value, which is returned, and only the TTL is refreshed."""
from redis.asyncio import Redis
_redis_client: Final[Redis] = self.init_async_client()
@ -1484,8 +1503,12 @@ class RedisCache(BaseCache):
_used_ttl: Final = self.get_ttl(ttl=ttl)
key = self.check_and_fix_namespace(key=key)
try:
result: Final = await self._incrbyfloat_with_ttl(
_redis_client, key=key, value=value, ttl=_used_ttl, refresh_ttl=refresh_ttl
result: Final = await (
self._set_if_missing_with_ttl(key=key, value=value, ttl=_used_ttl)
if only_if_missing
else self._incrbyfloat_with_ttl(
_redis_client, key=key, value=value, ttl=_used_ttl, refresh_ttl=refresh_ttl
)
)
## LOGGING ##

View file

@ -1932,6 +1932,7 @@ SPEND_LOG_QUEUE_POLL_INTERVAL: Final = get_env_float("SPEND_LOG_QUEUE_POLL_INTER
RESPONSES_SESSION_LOOKUP_MAX_ATTEMPTS: Final = max(1, get_env_int("RESPONSES_SESSION_LOOKUP_MAX_ATTEMPTS", 3))
RESPONSES_SESSION_LOOKUP_RETRY_INTERVAL: Final = get_env_float("RESPONSES_SESSION_LOOKUP_RETRY_INTERVAL", 0.2)
SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE: Final = get_env_int("SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE", 10000)
SPEND_COUNTER_COLD_SEED_JOIN_SECONDS: Final = 0.05
PROXY_DB_LOOKUP_MAX_CONCURRENCY: Final = max(1, get_env_int("PROXY_DB_LOOKUP_MAX_CONCURRENCY", 25))
PROXY_DB_LOOKUP_DEADLINE_SECONDS: Final = max(0.1, get_env_float("PROXY_DB_LOOKUP_DEADLINE_SECONDS", 10.0))
PROXY_DB_LOOKUP_STALL_WINDOW_SECONDS: Final = max(0.0, get_env_float("PROXY_DB_LOOKUP_STALL_WINDOW_SECONDS", 30.0))

View file

@ -13,22 +13,27 @@ memory in long-lived deployments.
"""
import asyncio
import functools
from collections import OrderedDict
from collections.abc import Mapping
from collections.abc import Awaitable, Callable, Mapping
from contextvars import ContextVar
from dataclasses import dataclass
from datetime import datetime, timezone
from types import MappingProxyType
from typing import TYPE_CHECKING, ClassVar, Final, Optional
from litellm._internal_context import with_service_target
from litellm._logging import verbose_proxy_logger
from litellm.constants import SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE
from litellm.constants import SPEND_COUNTER_COLD_SEED_JOIN_SECONDS, SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE
from litellm.litellm_core_utils.duration_parser import duration_in_seconds
from litellm.proxy._types import Litellm_EntityType
from litellm.proxy.db.db_lookup_gate import bounded_db_lookup, db_lookup_gate
from litellm.proxy.spend_tracking.spend_counter_batch import (
SPEND_COUNTERS_TARGET,
SpendCounterRead,
read_batched_spend_counter,
record_spend_counter_value,
spend_counter_tick,
)
from litellm.repositories.organization_repository import OrganizationRepository
from litellm.repositories.project_repository import ProjectRepository
@ -68,6 +73,58 @@ _WINDOW_SPEND_LOG_FIELDS: Final[Mapping[str, str]] = MappingProxyType(
)
@dataclass(slots=True)
class _CachedSpendSeed:
"""This process adding its cached spend to one cold counter: the add in flight, the tick it started at, the tick
the last one landed at, and the least the counter held when that one landed."""
started_at: int = 0
landed_at: int = 0
landed_spend: float = 0.0
in_flight: asyncio.Future[float | None] | None = None
def on_this_loop(self) -> bool:
return self.in_flight is None or self.in_flight.get_loop() is asyncio.get_running_loop()
async def run(self, add: Callable[[], Awaitable[float | None]]) -> float | None:
"""Run ``add`` in the calling task, so cancelling that task stops it, and publish its result to the requests
waiting on ``in_flight``, which ends cancelled when ``add`` fails or is cancelled."""
in_flight: Final[asyncio.Future[float | None]] = asyncio.get_running_loop().create_future()
self.started_at = spend_counter_tick()
self.in_flight = in_flight
try:
added: Final = await add()
self.landed_at = spend_counter_tick()
in_flight.set_result(added)
return added
finally:
if self.in_flight is in_flight:
self.in_flight = None
in_flight.cancel()
@functools.lru_cache(maxsize=SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE)
def _cached_spend_seed(spend_counter_cache: "DualCache", counter_key: str) -> _CachedSpendSeed:
return _CachedSpendSeed()
_cold_seed_waited: Final[ContextVar[float]] = ContextVar("cold_spend_seed_waited", default=0.0)
async def _wait_for_cold_seed(in_flight: asyncio.Future[float | None]) -> None:
"""Wait on ``in_flight`` for what this context has left of ``SPEND_COUNTER_COLD_SEED_JOIN_SECONDS``."""
loop: Final = asyncio.get_running_loop()
waited: Final = _cold_seed_waited.get()
started: Final = loop.time()
await asyncio.wait((in_flight,), timeout=max(0.0, SPEND_COUNTER_COLD_SEED_JOIN_SECONDS - waited))
_cold_seed_waited.set(waited + loop.time() - started)
def _redis_breaker_closed(spend_counter_cache: "DualCache") -> bool:
redis_cache: Final = spend_counter_cache.redis_cache
return redis_cache is None or redis_cache.circuit_breaker_closed
def _as_utc(value: datetime) -> datetime:
return value if value.tzinfo is not None else value.replace(tzinfo=timezone.utc)
@ -112,12 +169,71 @@ class SpendCounterReseed:
SpendCounterReseed._locks.popitem(last=False)
return lock
@staticmethod
async def add_cached_spend_once(
spend_counter_cache: "DualCache",
counter_key: str,
read: SpendCounterRead,
spend: float,
add: Callable[[], Awaitable[float | None]],
set_if_missing: Callable[[], Awaitable[float | None]],
) -> None:
"""Run ``add``, this process adding ``spend`` to a cold counter, once for all of this process's requests that
read the counter cold before that add landed. A request joins an add that started after its read came back and
gets its result, and starts or joins another add when that one fails or is cancelled. A request whose read went
out before an add landed, and one still waiting on an add after ``SPEND_COUNTER_COLD_SEED_JOIN_SECONDS`` across
its counters, runs ``set_if_missing`` in place of ``add``, then runs ``add`` too if the counter holds less than
the spend it should already hold. A request on another event loop than the add in flight, one that finds the
Redis circuit breaker not closed, and each other process, still run ``add``."""
seed: Final = _cached_spend_seed(spend_counter_cache, counter_key)
async def add_spend() -> float | None:
added: Final = await add()
seed.landed_spend = spend
return added
async def add_if_short() -> float | None:
least: Final = min(spend, seed.landed_spend) if seed.landed_at >= read.sent_at else spend
current: Final = await set_if_missing()
if current is not None and current < least:
return await add_spend()
seed.landed_spend = least
return current
async def add_unless_covered() -> float | None:
in_flight: Final = seed.in_flight
if not seed.on_this_loop() or not _redis_breaker_closed(spend_counter_cache):
return await add()
if in_flight is None:
return await seed.run(add_spend if seed.landed_at < read.sent_at else add_if_short)
covers: Final = seed.started_at > read.returned_at
await _wait_for_cold_seed(in_flight)
if not in_flight.done():
return await seed.run(add_if_short)
if covers and not in_flight.cancelled():
return in_flight.result()
return await add_unless_covered()
added: Final = await add_unless_covered()
if added is not None:
record_spend_counter_value(counter_key, added)
@staticmethod
@with_service_target(SPEND_COUNTERS_TARGET)
async def increment_in_memory(spend_counter_cache: "DualCache", counter_key: str, increment: float) -> float | None:
"""Apply local deltas after an in-flight reseed establishes the spend balance."""
async def increment_in_memory(
spend_counter_cache: "DualCache", counter_key: str, increment: float, only_if_missing: bool = False
) -> float | None:
"""Apply local deltas after an in-flight reseed establishes the spend balance. With ``only_if_missing``, a
counter that exists keeps its value, which is returned."""
lock: Final = await SpendCounterReseed._get_lock(counter_key)
async with lock:
current: Final[float | None] = ( # pyright: ignore[reportUnknownVariableType] # untyped in-memory cache
spend_counter_cache.in_memory_cache.get_cache(key=counter_key) # pyright: ignore[reportUnknownMemberType] # untyped in-memory cache
if only_if_missing
else None
)
if current is not None:
return float(current)
return await spend_counter_cache.async_increment_cache(
key=counter_key, value=increment, local_only=True, refresh_ttl=True
)

View file

@ -963,6 +963,7 @@ from litellm.proxy.spend_tracking.spend_counter_batch import (
read_batched_spend_counter,
record_spend_counter_value,
spend_counter_batch_scope,
spend_counter_read,
)
from litellm.proxy.spend_tracking.spend_management_endpoints import (
router as spend_management_router,
@ -4231,6 +4232,7 @@ async def ensure_spend_counter_initialized(
) -> None:
is_warm: Final = await _is_spend_counter_cache_warm(counter_key=counter_key)
if is_warm is False:
read: Final = spend_counter_read(counter_key)
# Shares the per-counter lock with get_current_spend.
db_spend: Final = await SpendCounterReseed.coalesced(
prisma_client=prisma_client,
@ -4242,7 +4244,16 @@ async def ensure_spend_counter_initialized(
# DB unavailable - fall back to in-process cache (may be stale).
base_spend: Final = await _get_source_cache_base_spend(source_cache_key=source_cache_key)
if base_spend > 0:
await increment_spend_counter_cache(counter_key=counter_key, increment=base_spend)
await SpendCounterReseed.add_cached_spend_once(
spend_counter_cache=spend_counter_cache,
counter_key=counter_key,
read=read,
spend=base_spend,
add=lambda: increment_spend_counter_cache(counter_key=counter_key, increment=base_spend),
set_if_missing=lambda: increment_spend_counter_cache(
counter_key=counter_key, increment=base_spend, only_if_missing=True
),
)
_ensure_spend_counter_initialized: Final = ensure_spend_counter_initialized
@ -4345,13 +4356,24 @@ async def refresh_spend_counter_ttl(counter_key: str) -> bool:
@with_service_target(SPEND_COUNTERS_TARGET)
async def increment_spend_counter_cache(counter_key: str, increment: float) -> float | None:
async def increment_spend_counter_cache(
counter_key: str, increment: float, only_if_missing: bool = False
) -> float | None:
if spend_counter_cache.redis_cache is not None:
try:
current_value: Final = await spend_counter_cache.redis_cache.async_increment(
key=counter_key,
value=increment,
refresh_ttl=True,
current_value: Final = await (
spend_counter_cache.redis_cache.async_increment(
key=counter_key,
value=increment,
refresh_ttl=True,
only_if_missing=True,
)
if only_if_missing
else spend_counter_cache.redis_cache.async_increment(
key=counter_key,
value=increment,
refresh_ttl=True,
)
)
except Exception:
await invalidate_spend_counter(counter_key=counter_key)
@ -4364,7 +4386,10 @@ async def increment_spend_counter_cache(counter_key: str, increment: float) -> f
return current_value
return await SpendCounterReseed.increment_in_memory(
spend_counter_cache=spend_counter_cache, counter_key=counter_key, increment=increment
spend_counter_cache=spend_counter_cache,
counter_key=counter_key,
increment=increment,
only_if_missing=only_if_missing,
)

View file

@ -1,11 +1,12 @@
"""One Redis MGET per phase (admission, reservation, post-call) for the spend counters it reads, not one GET each."""
import asyncio
import itertools
from collections.abc import Iterator, Mapping, Sequence
from contextvars import ContextVar, Token
from dataclasses import dataclass
from types import MappingProxyType, TracebackType
from typing import Final
from typing import Final, TypeAlias
from pydantic import TypeAdapter
@ -22,6 +23,24 @@ from litellm.proxy.common_utils.user_api_key_cache import (
_CounterValues: Final = TypeAdapter(dict[str, float | None])
_NO_VALUES: Final[Mapping[str, float | None]] = MappingProxyType({})
SPEND_COUNTERS_TARGET: Final = "spend_counters"
_ticks: Final = itertools.count(1)
def spend_counter_tick() -> int:
"""The next number in a per-process sequence that orders spend counter reads and cold seeds."""
return next(_ticks)
@dataclass(frozen=True, slots=True)
class SpendCounterRead:
"""When a read of a spend counter went out and when its reply came back, as spend counter ticks."""
sent_at: int
returned_at: int
_NO_READS: Final[Mapping[str, SpendCounterRead]] = MappingProxyType({})
_PendingMGet: TypeAlias = tuple[BatchResult[Mapping[str, object]], int]
@dataclass(frozen=True, slots=True)
@ -39,7 +58,17 @@ class SpendCounterBatch:
hook declares whatever is bound but unread, so whoever flushes first (the auth object prefetch, usually)
carries the spend counters in the same round trip."""
__slots__ = ("_fetched", "_inflight", "_keys", "_loaded", "_lock", "_open", "_redis_cache", "_request_batch")
__slots__ = (
"_fetched",
"_inflight",
"_keys",
"_loaded",
"_lock",
"_open",
"_reads",
"_redis_cache",
"_request_batch",
)
def __init__(self, redis_cache: RedisCache) -> None:
self._redis_cache: Final = redis_cache
@ -48,7 +77,8 @@ class SpendCounterBatch:
self._keys: frozenset[str] = frozenset()
self._fetched: frozenset[str] = frozenset()
self._loaded: Mapping[str, float | None] = _NO_VALUES
self._inflight: Final[list[BatchResult[Mapping[str, object]]]] = [] # mutable-ok: drained by _load
self._reads: Mapping[str, SpendCounterRead] = _NO_READS
self._inflight: Final[list[_PendingMGet]] = [] # mutable-ok: drained by _load
self._request_batch: Final[RedisBatch | None] = active_request_redis_batch(redis_cache)
if self._request_batch is not None:
self._request_batch.add_flush_hook(self._declare_pending)
@ -78,6 +108,10 @@ class SpendCounterBatch:
return None
return loaded[counter_key], True
def counter_read(self, counter_key: str) -> SpendCounterRead | None:
"""When the MGET behind this batch's value for the counter went out and came back, None without that value."""
return self._reads.get(counter_key) if self._open and counter_key in self._loaded else None
def record(self, counter_key: str, value: float) -> None:
"""A write returned the counter's new value; later reads in this scope see it instead of the MGET value."""
if not self._open:
@ -93,6 +127,7 @@ class SpendCounterBatch:
self._keys = self._keys - key
self._fetched = self._fetched - key
self._loaded = MappingProxyType({k: v for k, v in self._loaded.items() if k != counter_key})
self._reads = MappingProxyType({k: v for k, v in self._reads.items() if k != counter_key})
async def _load(self) -> Mapping[str, float | None]:
async with self._lock:
@ -103,10 +138,15 @@ class SpendCounterBatch:
pending: Final = self._keys - self._fetched
if pending:
self._fetched = self._fetched | pending
sent_at: Final = spend_counter_tick()
fetched: Final = await self._fetch(pending)
self._loaded = MappingProxyType({**fetched, **self._loaded})
self._merge(fetched, SpendCounterRead(sent_at=sent_at, returned_at=spend_counter_tick()))
return self._loaded
def _merge(self, fetched: Mapping[str, float | None], read: SpendCounterRead) -> None:
self._loaded = MappingProxyType({**fetched, **self._loaded})
self._reads = MappingProxyType({**dict.fromkeys(fetched, read), **self._reads})
def _declare_pending(self) -> None:
"""Flush hook: put every bound-but-unread counter on the request pipeline that is about to go out."""
if self._request_batch is None or not self._open:
@ -114,19 +154,21 @@ class SpendCounterBatch:
pending: Final = self._keys - self._fetched
if pending:
self._fetched = self._fetched | pending
sent_at: Final = spend_counter_tick()
with service_target(SPEND_COUNTERS_TARGET):
self._inflight.append(self._request_batch.mget(sorted(pending)))
read: Final = self._request_batch.mget(sorted(pending))
self._inflight.append((read, sent_at))
async def _collect_inflight(self) -> None:
results: Final = tuple(self._inflight)
reads: Final = tuple(self._inflight)
self._inflight.clear()
for result in results:
for read, sent_at in reads:
try:
fetched: Mapping[str, float | None] = _CounterValues.validate_python(await result)
fetched: Mapping[str, float | None] = _CounterValues.validate_python(await read)
except Exception as e: # noqa: BLE001 # per-key reads take over and apply their own Redis fallback
verbose_proxy_logger.debug("spend counter batch read failed, falling back to per-key reads: %s", e)
continue
self._loaded = MappingProxyType({**fetched, **self._loaded})
self._merge(fetched, SpendCounterRead(sent_at=sent_at, returned_at=spend_counter_tick()))
async def _fetch(self, keys: frozenset[str]) -> Mapping[str, float | None]:
try:
@ -145,6 +187,17 @@ def active_spend_counter_batch() -> SpendCounterBatch | None:
return _active_batch.get()
def spend_counter_read(counter_key: str) -> SpendCounterRead:
"""When this context's read of the counter went out and came back: the open batch's MGET for it, else now, since
``SpendCounterReseed.coalesced`` reads the counter again after a read outside the batch."""
batch: Final = _active_batch.get()
batched: Final = batch.counter_read(counter_key) if batch is not None else None
if batched is not None:
return batched
now: Final = spend_counter_tick()
return SpendCounterRead(sent_at=now, returned_at=now)
class spend_counter_batch_scope:
"""Reads inside the scope share one MGET for the keys bound here or by ``bind_*`` calls inside it.
Opened inside a scope whose batch is still open, it binds into that batch so both phases share the MGET."""

View file

@ -1,17 +1,20 @@
import json
import os
import re
import uuid
from collections.abc import Callable, Mapping
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass
from typing import Final
import httpx
import pytest
from integration._support.client import Gateway, Scenario, eventually, object_value, string_value
from integration._support.database import read_rows
from integration._support.database import read_rows, write_rows
from integration._support.upstream import delete_scenario, register_scenario
from integration.cost_calculation.cost_tracking_case import JsonResponse
from pydantic import BaseModel, JsonValue
from redis import Redis
PROVIDER_AUTHORIZATION: Final = "Bearer integration-provider-key"
CANNED_CONTENT: Final = "Hello! This is a mock response from the fake OpenAI endpoint."
@ -649,3 +652,141 @@ def test_streamed_call_is_refused_as_json_once_over_budget(gateway: Gateway) ->
assert "data:" not in denied.text, denied.text
_assert_budget_refusal(denied, end_user, 0.05)
assert _observed_requests(upstream) == []
_COLD_BURST: Final = 40
_COLD_SEED_SPEND: Final = 6.0
_COLD_BUDGET: Final = 10.0
_CHAT_COST: Final = 20 * 0.001 + 20 * 0.002
def _serves_model(gateway: Gateway, model: str) -> bool:
response: Final = gateway.request("GET", "/model/info")
return response.status_code == 200 and any(
object_value(entry)["model_name"] == model for entry in object_value(response.json())["data"]
)
def _counter_cache() -> Redis:
return Redis(host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"]))
def _counter(cache: Redis, end_user: str) -> float | None:
raw: Final = cache.get(f"spend:end_user:{end_user}")
return None if raw is None else float(raw)
def _lose_counter(cache: Redis, end_user: str) -> None:
cache.delete(f"spend:end_user:{end_user}")
def _set_end_user_row_spend(end_user: str, spend: float) -> None:
write_rows('UPDATE "LiteLLM_EndUserTable" SET spend = %s WHERE user_id = %s', (str(spend), end_user))
def _await_row_spend(end_user: str, spend: float) -> None:
eventually(
lambda: _end_user_row(end_user),
lambda rows: len(rows) == 1 and float(str(rows[0]["spend"])) == pytest.approx(spend),
seconds=70,
)
def _refused_spend(denied: httpx.Response, end_user: str) -> float:
error: Final = _denial_error(denied)
assert error["type"] == "budget_exceeded", denied.text
matched: Final = _REFUSAL_MESSAGE.match(string_value(error["message"]))
assert matched is not None, denied.text
assert matched.group("user") == end_user
return float(matched.group("spend"))
@pytest.mark.parametrize("spelling", ("user", "responses_user", "messages_metadata_user_id"))
def test_overlapping_cold_starts_count_the_end_user_spend_once(gateway: Gateway, spelling: str) -> None:
chosen: Final = SPELLINGS[spelling]
with (
gateway.scenario() as scenario,
httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream,
_counter_cache() as cache,
):
deployment: Final = _deployment(scenario, chosen)
key: Final = scenario.key(models=[deployment.model])
end_user: Final = f"integration-end-user-{uuid.uuid4().hex}"
_new_customer(gateway, end_user, max_budget=_COLD_BUDGET, spend=_COLD_SEED_SPEND)
scenario.cleanups.callback(gateway.request, "POST", "/customer/delete", {"user_ids": [end_user]})
assert _customer_info(gateway, end_user).spend == _COLD_SEED_SPEND
assert _counter(cache, end_user) is None
upstream.delete("/__observations").raise_for_status()
with ThreadPoolExecutor(max_workers=_COLD_BURST) as pool:
burst: Final = tuple(
pool.map(
lambda index: _send(gateway, chosen, deployment, key, end_user, f"cold {index}"),
range(_COLD_BURST),
)
)
refused: Final = [response.text for response in burst if response.status_code != 200]
assert refused == [], refused
follow_up: Final = _send(gateway, chosen, deployment, key, end_user, "cold follow-up")
_assert_served(chosen, follow_up)
assert len(_observed_requests(upstream)) == _COLD_BURST + 1
expected_spend: Final = _COLD_SEED_SPEND + (_COLD_BURST + 1) * _CHAT_COST
_await_row_spend(end_user, expected_spend)
assert _customer_info(gateway, end_user).spend == pytest.approx(expected_spend)
counted: Final = eventually(
lambda: _counter(cache, end_user),
lambda value: value is not None and value == pytest.approx(expected_spend),
seconds=30,
return_last_on_timeout=True,
)
assert counted == pytest.approx(expected_spend)
def test_a_lost_counter_keeps_spend_this_proxy_has_not_written_yet(gateway: Gateway) -> None:
with gateway.scenario() as scenario, _counter_cache() as cache:
model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.25)
key: Final = scenario.key(models=[model])
end_user: Final = f"integration-end-user-{uuid.uuid4().hex}"
call_cost: Final = 20 * 0.001 + 20 * 0.25
_new_customer(gateway, end_user, max_budget=_COLD_BUDGET, spend=_COLD_SEED_SPEND)
scenario.cleanups.callback(gateway.request, "POST", "/customer/delete", {"user_ids": [end_user]})
served: Final = _chat(gateway, model, key, end_user)
assert served.status_code == 200, served.text
_await_row_spend(end_user, _COLD_SEED_SPEND + call_cost)
_set_end_user_row_spend(end_user, _COLD_SEED_SPEND)
_lose_counter(cache, end_user)
denied: Final = _chat(gateway, model, key, end_user)
assert _refused_spend(denied, end_user) == pytest.approx(_COLD_SEED_SPEND + call_cost)
def test_a_lost_counter_keeps_spend_another_proxy_has_not_written_yet(gateway: Gateway, peer: Gateway) -> None:
with gateway.scenario() as scenario, _counter_cache() as cache:
model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.149)
eventually(lambda: _serves_model(peer, model), bool, seconds=30)
key: Final = scenario.key(models=[model])
end_user: Final = f"integration-end-user-{uuid.uuid4().hex}"
call_cost: Final = 20 * 0.001 + 20 * 0.149
_new_customer(gateway, end_user, max_budget=_COLD_BUDGET, spend=_COLD_SEED_SPEND)
scenario.cleanups.callback(gateway.request, "POST", "/customer/delete", {"user_ids": [end_user]})
first: Final = _chat(peer, model, key, end_user)
assert first.status_code == 200, first.text
_await_row_spend(end_user, _COLD_SEED_SPEND + call_cost)
_set_end_user_row_spend(end_user, _COLD_SEED_SPEND)
_lose_counter(cache, end_user)
second: Final = _chat(peer, model, key, end_user)
assert second.status_code == 200, second.text
expected_spend: Final = _COLD_SEED_SPEND + 2 * call_cost
counted: Final = eventually(
lambda: _counter(cache, end_user),
lambda value: value is not None and value == pytest.approx(expected_spend),
seconds=30,
return_last_on_timeout=True,
)
denied: Final = _chat(gateway, model, key, end_user)
assert counted == pytest.approx(expected_spend)
assert _denial_error(denied)["type"] == "budget_exceeded", denied.text

View file

@ -8,6 +8,7 @@ from typing import Final, cast
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from fakeredis import FakeAsyncRedis, FakeServer
import litellm
from litellm._service_logger import ServiceLogging
@ -1474,6 +1475,55 @@ async def test_async_increment_pipelines_the_ttl_command(
assert client.round_trips == expected_round_trips
@pytest.mark.asyncio
@pytest.mark.parametrize(
("existing", "expected"),
[
pytest.param(None, 1.5, id="missing key: set to the value"),
pytest.param(0.25, 0.25, id="key below the value: kept"),
pytest.param(4.0, 4.0, id="key above the value: kept"),
],
)
async def test_async_increment_only_if_missing_sets_a_missing_key_and_refreshes_the_ttl(
monkeypatch, redis_no_ping, existing, expected
):
monkeypatch.setenv("REDIS_HOST", "https://my-test-host")
redis_cache = RedisCache(namespace="ns")
client = FakeAsyncRedis(server=FakeServer())
if existing is not None:
await client.set("ns:spend:key:k", existing, ex=5)
with patch.object(redis_cache, "init_async_client", return_value=client):
current = await redis_cache.async_increment(
key="spend:key:k", value=1.5, ttl=60, refresh_ttl=True, only_if_missing=True
)
after_add = await redis_cache.async_increment(key="spend:key:k", value=2.0, ttl=60, refresh_ttl=True)
assert current == expected
assert after_add == expected + 2.0
assert await client.ttl("ns:spend:key:k") == 60
@pytest.mark.asyncio
async def test_async_increment_only_if_missing_reports_a_redis_failure_like_an_increment(monkeypatch, redis_no_ping):
monkeypatch.setenv("REDIS_HOST", "https://my-test-host")
redis_cache = RedisCache(namespace="ns")
client = MagicMock()
client.pipeline.side_effect = ConnectionError("connection reset")
failure_hook = AsyncMock()
with (
patch.object(redis_cache, "init_async_client", return_value=client),
patch.object(redis_cache.service_logger_obj, "async_service_failure_hook", failure_hook),
pytest.raises(ConnectionError),
):
await redis_cache.async_increment(key="spend:key:k", value=1.5, ttl=60, refresh_ttl=True, only_if_missing=True)
await asyncio.sleep(0)
assert failure_hook.await_args.kwargs["call_type"] == "async_increment"
assert redis_cache._circuit_breaker._hard_failure_count == 1
class _SetRecordingPipeline:
def __init__(self) -> None:
self.sets: list[tuple[str, str, timedelta | None]] = []

File diff suppressed because it is too large Load diff