refactor(redis-store): move session budgets behind a store

Co-authored-by: Yujong Lee <yujong@berri.ai>
This commit is contained in:
Devin AI 2026-10-30 16:15:42 +00:00
parent 8efade866f
commit 61d8a0b164
4 changed files with 163 additions and 105 deletions

View file

@ -14,8 +14,8 @@ Works across multiple proxy instances via DualCache (in-memory + Redis).
Follows the same pattern as max_iterations_limiter.py.
"""
import logging
import os
from datetime import timedelta
from typing import TYPE_CHECKING, Any, Final
from pydantic import ConfigDict, TypeAdapter
@ -23,12 +23,13 @@ from pydantic import ConfigDict, TypeAdapter
from litellm import DualCache
from litellm._internal_context import with_service_target
from litellm._logging import verbose_proxy_logger
from litellm.caching.redis_cache import log_redis_failure
from litellm.exceptions import RateLimitType
from litellm.integrations.custom_logger import CustomLogger
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
from litellm.proxy.hooks.rate_limiter_utils import resolve_llm_provider_for_rate_limit
from litellm.redis.stores.session_budget import SessionBudgetStore, TransportSessionBudgetStore, spend_key
from litellm.redis.transport.contract import RedisFailure
if TYPE_CHECKING:
from litellm.proxy.utils import InternalUsageCache as _InternalUsageCache
@ -38,31 +39,10 @@ else:
InternalUsageCache = Any
_CACHE_AMOUNT_ADAPTER: Final[TypeAdapter[str | bytes | int | float]] = TypeAdapter(
str | bytes | int | float, config=ConfigDict(strict=True, hide_input_in_errors=True)
)
_OPTIONAL_CACHE_AMOUNT_ADAPTER: Final[TypeAdapter[str | bytes | int | float | None]] = TypeAdapter(
str | bytes | int | float | None, config=ConfigDict(strict=True, hide_input_in_errors=True)
)
# Redis Lua script for atomic float increment with TTL.
# INCRBYFLOAT returns the new value as a string.
# Only sets EXPIRE on first call (when prior value was nil).
MAX_BUDGET_SESSION_INCREMENT_SCRIPT: Final = """
local key = KEYS[1]
local amount = ARGV[1]
local ttl = tonumber(ARGV[2])
local existed = redis.call('EXISTS', key)
local new_val = redis.call('INCRBYFLOAT', key, amount)
if existed == 0 then
redis.call('EXPIRE', key, ttl)
end
return new_val
"""
# Default TTL for session budget counters (1 hour)
DEFAULT_MAX_BUDGET_PER_SESSION_TTL: Final = 3600
@ -87,12 +67,10 @@ class _PROXY_MaxBudgetPerSessionHandler(CustomLogger):
)
)
if self.internal_usage_cache.dual_cache.redis_cache is not None:
self.increment_script = self.internal_usage_cache.dual_cache.redis_cache.async_register_script(
MAX_BUDGET_SESSION_INCREMENT_SCRIPT
)
else:
self.increment_script = None
redis_cache: Final = self.internal_usage_cache.dual_cache.redis_cache
self._store: Final[SessionBudgetStore | None] = (
TransportSessionBudgetStore(redis_cache.transport()) if redis_cache is not None else None
)
@with_service_target("session_budgets")
async def async_pre_call_hook(
@ -114,8 +92,7 @@ class _PROXY_MaxBudgetPerSessionHandler(CustomLogger):
return None
max_budget = float(max_budget)
cache_key: Final = self._make_cache_key(session_id)
current_spend: Final = await self._get_current_spend(cache_key)
current_spend: Final = await self._get_current_spend(session_id)
verbose_proxy_logger.debug(
"MaxBudgetPerSessionHandler: session_id=%s, spend=%.4f, max=%.2f",
@ -172,8 +149,7 @@ class _PROXY_MaxBudgetPerSessionHandler(CustomLogger):
if response_cost <= 0:
return
cache_key: Final = self._make_cache_key(str(session_id))
await self._increment_spend(cache_key, float(response_cost))
await self._increment_spend(str(session_id), float(response_cost))
verbose_proxy_logger.debug(
"MaxBudgetPerSessionHandler: incremented session %s spend by %.6f",
@ -219,30 +195,19 @@ class _PROXY_MaxBudgetPerSessionHandler(CustomLogger):
return None
def _make_cache_key(self, session_id: str) -> str:
return f"{{session_budget:{session_id}}}:spend"
return spend_key(session_id)
@with_service_target("session_budgets")
async def _get_current_spend(self, cache_key: str) -> float:
async def _get_current_spend(self, session_id: str) -> float:
"""Read current accumulated spend for a session."""
if self.internal_usage_cache.dual_cache.redis_cache is not None:
try:
redis_result: Final[str | bytes | int | float | None] = _OPTIONAL_CACHE_AMOUNT_ADAPTER.validate_python(
await self.internal_usage_cache.dual_cache.redis_cache.async_get_cache(key=cache_key)
)
if redis_result is not None:
return float(redis_result)
return 0.0
except Exception as e:
log_redis_failure(
verbose_proxy_logger,
logging.WARNING,
"MaxBudgetPerSessionHandler: Redis GET failed, falling back to in-memory",
e,
)
if self._store is not None:
stored: Final = await self._store.spend(session_id)
if not isinstance(stored, RedisFailure):
return 0.0 if stored is None else stored
cache_result: Final[str | bytes | int | float | None] = _OPTIONAL_CACHE_AMOUNT_ADAPTER.validate_python(
await self.internal_usage_cache.async_get_cache(
key=cache_key,
key=self._make_cache_key(session_id),
litellm_parent_otel_span=None,
local_only=True,
)
@ -251,26 +216,14 @@ class _PROXY_MaxBudgetPerSessionHandler(CustomLogger):
return float(cache_result)
return 0.0
async def _increment_spend(self, cache_key: str, amount: float) -> float:
async def _increment_spend(self, session_id: str, amount: float) -> float:
"""Atomically increment the session spend and return the new value."""
if self.increment_script is not None:
try:
script_result: Final[str | bytes | int | float] = _CACHE_AMOUNT_ADAPTER.validate_python(
await self.increment_script(
keys=[cache_key],
args=[str(amount), self.ttl],
)
)
return float(script_result)
except Exception as e:
log_redis_failure(
verbose_proxy_logger,
logging.WARNING,
"MaxBudgetPerSessionHandler: Redis INCRBYFLOAT failed, falling back to in-memory",
e,
)
if self._store is not None:
added: Final = await self._store.add_spend(session_id, amount, ttl=timedelta(seconds=self.ttl))
if not isinstance(added, RedisFailure):
return added
return await self._in_memory_increment_spend(cache_key, amount)
return await self._in_memory_increment_spend(self._make_cache_key(session_id), amount)
async def _in_memory_increment_spend(self, cache_key: str, amount: float) -> float:
current: Final[str | bytes | int | float | None] = _OPTIONAL_CACHE_AMOUNT_ADAPTER.validate_python(

View file

@ -0,0 +1,55 @@
from dataclasses import dataclass
from datetime import timedelta
from typing import Final, Protocol
from litellm.redis.transport.contract import EvalReply, RedisFailure, RedisTransport
_ADD_SPEND: Final = """
local key = KEYS[1]
local amount = ARGV[1]
local ttl = tonumber(ARGV[2])
local existed = redis.call('EXISTS', key)
local new_val = redis.call('INCRBYFLOAT', key, amount)
if existed == 0 then
redis.call('EXPIRE', key, ttl)
end
return new_val
"""
def spend_key(session_id: str) -> str:
return f"{{session_budget:{session_id}}}:spend"
class SessionBudgetStore(Protocol):
async def add_spend(self, session_id: str, amount: float, *, ttl: timedelta) -> float | RedisFailure: ...
async def spend(self, session_id: str) -> float | None | RedisFailure: ...
@dataclass(frozen=True, slots=True)
class TransportSessionBudgetStore:
transport: RedisTransport
async def add_spend(self, session_id: str, amount: float, *, ttl: timedelta) -> float | RedisFailure:
reply: Final = await self.transport.eval(
_ADD_SPEND, (spend_key(session_id),), (str(amount), ttl // timedelta(seconds=1))
)
return reply if isinstance(reply, RedisFailure) else _amount(reply)
async def spend(self, session_id: str) -> float | None | RedisFailure:
raw: Final = await self.transport.get(spend_key(session_id))
if raw is None or isinstance(raw, RedisFailure):
return raw
return _amount(raw)
def _amount(reply: EvalReply) -> float | RedisFailure:
if not isinstance(reply, (bytes, int)):
return RedisFailure(kind="decode", message=f"session spend is not a number: {reply!r}")
try:
return float(reply)
except ValueError:
return RedisFailure(kind="decode", message=f"session spend is not a number: {reply!r}")

View file

@ -8,20 +8,22 @@ Tests that session-scoped budget tracking works correctly:
- Requests without agent_id pass through
"""
import logging
from unittest.mock import patch
import logging
import fakeredis
import pytest
from fastapi import HTTPException
from litellm.caching.caching import DualCache
from litellm.caching.redis_cache import _redis_circuit_breaker_guard
from litellm.caching.redis_cache import RedisCache
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.hooks.max_budget_per_session_limiter import (
PROXY_MaxBudgetPerSessionHandler,
)
from litellm.proxy.utils import InternalUsageCache
from litellm.types.agents import AgentResponse
from tests._support.fake_redis import fake_redis_cache
def _make_mock_agent(max_budget_per_session: float) -> AgentResponse:
@ -49,9 +51,7 @@ async def test_budget_per_session_under_budget_passes():
mock_agent = _make_mock_agent(max_budget_per_session=5.0)
with patch(
"litellm.proxy.agent_endpoints.agent_registry.global_agent_registry"
) as mock_registry:
with patch("litellm.proxy.agent_endpoints.agent_registry.global_agent_registry") as mock_registry:
mock_registry.get_agent_by_id.return_value = mock_agent
result = await handler.async_pre_call_hook(
@ -79,14 +79,11 @@ async def test_budget_per_session_exceeds_budget():
)
session_id = "session-over-budget"
cache_key = handler._make_cache_key(session_id)
await handler._increment_spend(cache_key, 1.50)
await handler._increment_spend(session_id, 1.50)
mock_agent = _make_mock_agent(max_budget_per_session=1.0)
with patch(
"litellm.proxy.agent_endpoints.agent_registry.global_agent_registry"
) as mock_registry:
with patch("litellm.proxy.agent_endpoints.agent_registry.global_agent_registry") as mock_registry:
mock_registry.get_agent_by_id.return_value = mock_agent
with pytest.raises(HTTPException) as exc_info:
@ -115,14 +112,11 @@ async def test_budget_per_session_independent_sessions():
agent_id="agent-budget-123",
)
cache_key_a = handler._make_cache_key("session-A")
await handler._increment_spend(cache_key_a, 3.0)
await handler._increment_spend("session-A", 3.0)
mock_agent = _make_mock_agent(max_budget_per_session=2.0)
with patch(
"litellm.proxy.agent_endpoints.agent_registry.global_agent_registry"
) as mock_registry:
with patch("litellm.proxy.agent_endpoints.agent_registry.global_agent_registry") as mock_registry:
mock_registry.get_agent_by_id.return_value = mock_agent
# Session A should be blocked
@ -167,34 +161,81 @@ async def test_no_agent_id_passes():
assert result is None
class _OpenBreakerRedis:
def __init__(self) -> None:
from litellm.caching.redis_cache import RedisCircuitBreaker
def _redis_handler(server: fakeredis.FakeServer, namespace: str | None = None) -> tuple[RedisCache, object]:
cache = fake_redis_cache(server, namespace=namespace)
return cache, PROXY_MaxBudgetPerSessionHandler(
internal_usage_cache=InternalUsageCache(DualCache(redis_cache=cache))
)
self._circuit_breaker = RedisCircuitBreaker(failure_threshold=3, recovery_timeout=60)
for _ in range(3):
self._circuit_breaker.record_failure()
@_redis_circuit_breaker_guard
async def async_get_cache(self, key, **kwargs):
raise AssertionError("never reached")
async def _pre_call(handler, session_id: str, max_budget: float) -> None:
with patch("litellm.proxy.agent_endpoints.agent_registry.global_agent_registry") as mock_registry:
mock_registry.get_agent_by_id.return_value = _make_mock_agent(max_budget_per_session=max_budget)
await handler.async_pre_call_hook(
user_api_key_dict=UserAPIKeyAuth(api_key="sk-test-key-budget", agent_id="agent-budget-123"),
cache=DualCache(),
data={"metadata": {"session_id": session_id}},
call_type="",
)
def async_register_script(self, script):
@_redis_circuit_breaker_guard
async def refused(_self, keys, args):
raise AssertionError("never reached")
return lambda keys, args: refused(self, keys, args)
@pytest.mark.asyncio
async def test_spend_written_through_redis_is_shared_with_old_pods_and_expires():
server = fakeredis.FakeServer()
cache, handler = _redis_handler(server, namespace="ns")
raw = fakeredis.FakeAsyncRedis(server=server)
assert await handler._increment_spend("s1", 1.25) == 1.25
assert await handler._increment_spend("s1", 0.5) == 1.75
assert await cache.async_get_cache(key="{session_budget:s1}:spend") == 1.75
assert 3_590_000 < await raw.pttl("ns:{session_budget:s1}:spend") <= 3_600_000
with pytest.raises(HTTPException):
await _pre_call(handler, "s1", max_budget=1.5)
@pytest.mark.asyncio
async def test_spend_an_old_pod_wrote_blocks_the_session():
server = fakeredis.FakeServer()
_, handler = _redis_handler(server, namespace="ns")
await fakeredis.FakeAsyncRedis(server=server).incrbyfloat("ns:{session_budget:s1}:spend", 2.5)
assert await handler._get_current_spend("s1") == 2.5
await _pre_call(handler, "s1", max_budget=3.0)
with pytest.raises(HTTPException):
await _pre_call(handler, "s1", max_budget=2.5)
@pytest.mark.asyncio
async def test_a_spend_that_does_not_decode_falls_back_to_local_spend():
server = fakeredis.FakeServer()
_, handler = _redis_handler(server)
await fakeredis.FakeAsyncRedis(server=server).set("{session_budget:s1}:spend", "not-a-number")
assert await handler._get_current_spend("s1") == 0.0
@pytest.mark.asyncio
async def test_an_unreachable_redis_keeps_counting_spend_locally():
server = fakeredis.FakeServer()
_, handler = _redis_handler(server)
server.connected = False
assert await handler._increment_spend("s1", 1.0) == 1.0
assert await handler._increment_spend("s1", 2.0) == 3.0
assert await handler._get_current_spend("s1") == 3.0
@pytest.mark.asyncio
async def test_an_open_circuit_breaker_reads_session_spend_locally_without_a_warning(caplog):
cache = DualCache(redis_cache=_OpenBreakerRedis()) # pyright: ignore[reportArgumentType] # duck-typed Redis double
handler = PROXY_MaxBudgetPerSessionHandler(internal_usage_cache=InternalUsageCache(cache))
server = fakeredis.FakeServer()
cache, handler = _redis_handler(server)
for _ in range(cache._circuit_breaker.failure_threshold):
cache._circuit_breaker.record_failure()
caplog.clear()
with caplog.at_level(logging.DEBUG, logger="LiteLLM Proxy"):
spend = await handler._get_current_spend("{session_budget:quiet}:spend")
with caplog.at_level(logging.DEBUG):
spend = await handler._get_current_spend("quiet")
assert spend == 0.0
assert [record.getMessage() for record in caplog.records if record.levelno >= logging.WARNING] == []

View file

@ -18,6 +18,9 @@ the new behavior:
internal modules.
"""
from typing import Final
import fakeredis
import pytest
from fastapi import HTTPException
@ -831,16 +834,22 @@ class TestProxyHooksActuallyRaiseProxyRateLimitError:
):
"""Drive `_PROXY_MaxBudgetPerSessionHandler` past its budget and
assert the unified class is raised."""
from unittest.mock import AsyncMock, MagicMock, patch
from unittest.mock import MagicMock, patch
from litellm.caching.caching import DualCache
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.hooks.max_budget_per_session_limiter import (
PROXY_MaxBudgetPerSessionHandler,
)
from litellm.proxy.utils import InternalUsageCache
from tests._support.fake_redis import fake_redis_cache
internal_cache = MagicMock()
internal_cache.async_get_cache = AsyncMock(return_value=10.0)
internal_cache: Final = InternalUsageCache(DualCache(redis_cache=fake_redis_cache(fakeredis.FakeServer())))
await internal_cache.async_set_cache(
key="{session_budget:session-over-budget}:spend",
value=10.0,
litellm_parent_otel_span=None,
)
handler = PROXY_MaxBudgetPerSessionHandler(
internal_usage_cache=internal_cache,
)