From 5eccb361cec0ce99cd573c95793566be0aa35e34 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 30 Oct 2026 16:15:42 +0000 Subject: [PATCH] refactor(redis-store): move session budgets behind a store Co-authored-by: Yujong Lee --- .../hooks/max_budget_per_session_limiter.py | 91 ++++----------- litellm/redis/stores/session_budget.py | 55 +++++++++ .../test_max_budget_per_session_limiter.py | 107 ++++++++++++------ .../unit/test_rate_limit_error_unification.py | 15 ++- 4 files changed, 163 insertions(+), 105 deletions(-) create mode 100644 litellm/redis/stores/session_budget.py diff --git a/litellm/proxy/hooks/max_budget_per_session_limiter.py b/litellm/proxy/hooks/max_budget_per_session_limiter.py index 002aeb145a5..1de1c58148d 100644 --- a/litellm/proxy/hooks/max_budget_per_session_limiter.py +++ b/litellm/proxy/hooks/max_budget_per_session_limiter.py @@ -14,8 +14,8 @@ Works across multiple proxy instances via DualCache (in-memory + Redis). Follows the same pattern as max_iterations_limiter.py. """ -import logging import os +from datetime import timedelta from typing import TYPE_CHECKING, Any, Final from pydantic import ConfigDict, TypeAdapter @@ -23,12 +23,13 @@ from pydantic import ConfigDict, TypeAdapter from litellm import DualCache from litellm._internal_context import with_service_target from litellm._logging import verbose_proxy_logger -from litellm.caching.redis_cache import log_redis_failure from litellm.exceptions import RateLimitType from litellm.integrations.custom_logger import CustomLogger from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError from litellm.proxy.hooks.rate_limiter_utils import resolve_llm_provider_for_rate_limit +from litellm.redis.stores.session_budget import SessionBudgetStore, TransportSessionBudgetStore, spend_key +from litellm.redis.transport.contract import RedisFailure if TYPE_CHECKING: from litellm.proxy.utils import InternalUsageCache as _InternalUsageCache @@ -38,31 +39,10 @@ else: InternalUsageCache = Any -_CACHE_AMOUNT_ADAPTER: Final[TypeAdapter[str | bytes | int | float]] = TypeAdapter( - str | bytes | int | float, config=ConfigDict(strict=True, hide_input_in_errors=True) -) _OPTIONAL_CACHE_AMOUNT_ADAPTER: Final[TypeAdapter[str | bytes | int | float | None]] = TypeAdapter( str | bytes | int | float | None, config=ConfigDict(strict=True, hide_input_in_errors=True) ) - -# Redis Lua script for atomic float increment with TTL. -# INCRBYFLOAT returns the new value as a string. -# Only sets EXPIRE on first call (when prior value was nil). -MAX_BUDGET_SESSION_INCREMENT_SCRIPT: Final = """ -local key = KEYS[1] -local amount = ARGV[1] -local ttl = tonumber(ARGV[2]) - -local existed = redis.call('EXISTS', key) -local new_val = redis.call('INCRBYFLOAT', key, amount) -if existed == 0 then - redis.call('EXPIRE', key, ttl) -end - -return new_val -""" - # Default TTL for session budget counters (1 hour) DEFAULT_MAX_BUDGET_PER_SESSION_TTL: Final = 3600 @@ -87,12 +67,10 @@ class _PROXY_MaxBudgetPerSessionHandler(CustomLogger): ) ) - if self.internal_usage_cache.dual_cache.redis_cache is not None: - self.increment_script = self.internal_usage_cache.dual_cache.redis_cache.async_register_script( - MAX_BUDGET_SESSION_INCREMENT_SCRIPT - ) - else: - self.increment_script = None + redis_cache: Final = self.internal_usage_cache.dual_cache.redis_cache + self._store: Final[SessionBudgetStore | None] = ( + TransportSessionBudgetStore(redis_cache.transport()) if redis_cache is not None else None + ) @with_service_target("session_budgets") async def async_pre_call_hook( @@ -114,8 +92,7 @@ class _PROXY_MaxBudgetPerSessionHandler(CustomLogger): return None max_budget = float(max_budget) - cache_key: Final = self._make_cache_key(session_id) - current_spend: Final = await self._get_current_spend(cache_key) + current_spend: Final = await self._get_current_spend(session_id) verbose_proxy_logger.debug( "MaxBudgetPerSessionHandler: session_id=%s, spend=%.4f, max=%.2f", @@ -172,8 +149,7 @@ class _PROXY_MaxBudgetPerSessionHandler(CustomLogger): if response_cost <= 0: return - cache_key: Final = self._make_cache_key(str(session_id)) - await self._increment_spend(cache_key, float(response_cost)) + await self._increment_spend(str(session_id), float(response_cost)) verbose_proxy_logger.debug( "MaxBudgetPerSessionHandler: incremented session %s spend by %.6f", @@ -219,30 +195,19 @@ class _PROXY_MaxBudgetPerSessionHandler(CustomLogger): return None def _make_cache_key(self, session_id: str) -> str: - return f"{{session_budget:{session_id}}}:spend" + return spend_key(session_id) @with_service_target("session_budgets") - async def _get_current_spend(self, cache_key: str) -> float: + async def _get_current_spend(self, session_id: str) -> float: """Read current accumulated spend for a session.""" - if self.internal_usage_cache.dual_cache.redis_cache is not None: - try: - redis_result: Final[str | bytes | int | float | None] = _OPTIONAL_CACHE_AMOUNT_ADAPTER.validate_python( - await self.internal_usage_cache.dual_cache.redis_cache.async_get_cache(key=cache_key) - ) - if redis_result is not None: - return float(redis_result) - return 0.0 - except Exception as e: - log_redis_failure( - verbose_proxy_logger, - logging.WARNING, - "MaxBudgetPerSessionHandler: Redis GET failed, falling back to in-memory", - e, - ) + if self._store is not None: + stored: Final = await self._store.spend(session_id) + if not isinstance(stored, RedisFailure): + return 0.0 if stored is None else stored cache_result: Final[str | bytes | int | float | None] = _OPTIONAL_CACHE_AMOUNT_ADAPTER.validate_python( await self.internal_usage_cache.async_get_cache( - key=cache_key, + key=self._make_cache_key(session_id), litellm_parent_otel_span=None, local_only=True, ) @@ -251,26 +216,14 @@ class _PROXY_MaxBudgetPerSessionHandler(CustomLogger): return float(cache_result) return 0.0 - async def _increment_spend(self, cache_key: str, amount: float) -> float: + async def _increment_spend(self, session_id: str, amount: float) -> float: """Atomically increment the session spend and return the new value.""" - if self.increment_script is not None: - try: - script_result: Final[str | bytes | int | float] = _CACHE_AMOUNT_ADAPTER.validate_python( - await self.increment_script( - keys=[cache_key], - args=[str(amount), self.ttl], - ) - ) - return float(script_result) - except Exception as e: - log_redis_failure( - verbose_proxy_logger, - logging.WARNING, - "MaxBudgetPerSessionHandler: Redis INCRBYFLOAT failed, falling back to in-memory", - e, - ) + if self._store is not None: + added: Final = await self._store.add_spend(session_id, amount, ttl=timedelta(seconds=self.ttl)) + if not isinstance(added, RedisFailure): + return added - return await self._in_memory_increment_spend(cache_key, amount) + return await self._in_memory_increment_spend(self._make_cache_key(session_id), amount) async def _in_memory_increment_spend(self, cache_key: str, amount: float) -> float: current: Final[str | bytes | int | float | None] = _OPTIONAL_CACHE_AMOUNT_ADAPTER.validate_python( diff --git a/litellm/redis/stores/session_budget.py b/litellm/redis/stores/session_budget.py new file mode 100644 index 00000000000..aae30fde8dd --- /dev/null +++ b/litellm/redis/stores/session_budget.py @@ -0,0 +1,55 @@ +from dataclasses import dataclass +from datetime import timedelta +from typing import Final, Protocol + +from litellm.redis.transport.contract import EvalReply, RedisFailure, RedisTransport + +_ADD_SPEND: Final = """ +local key = KEYS[1] +local amount = ARGV[1] +local ttl = tonumber(ARGV[2]) + +local existed = redis.call('EXISTS', key) +local new_val = redis.call('INCRBYFLOAT', key, amount) +if existed == 0 then + redis.call('EXPIRE', key, ttl) +end + +return new_val +""" + + +def spend_key(session_id: str) -> str: + return f"{{session_budget:{session_id}}}:spend" + + +class SessionBudgetStore(Protocol): + async def add_spend(self, session_id: str, amount: float, *, ttl: timedelta) -> float | RedisFailure: ... + + async def spend(self, session_id: str) -> float | None | RedisFailure: ... + + +@dataclass(frozen=True, slots=True) +class TransportSessionBudgetStore: + transport: RedisTransport + + async def add_spend(self, session_id: str, amount: float, *, ttl: timedelta) -> float | RedisFailure: + reply: Final = await self.transport.eval( + _ADD_SPEND, (spend_key(session_id),), (str(amount), ttl // timedelta(seconds=1)) + ) + return reply if isinstance(reply, RedisFailure) else _amount(reply) + + async def spend(self, session_id: str) -> float | None | RedisFailure: + raw: Final = await self.transport.get(spend_key(session_id)) + if raw is None or isinstance(raw, RedisFailure): + return raw + return _amount(raw) + + +def _amount(reply: EvalReply) -> float | RedisFailure: + if not isinstance(reply, (bytes, int)): + return RedisFailure(kind="decode", message=f"session spend is not a number: {reply!r}") + try: + return float(reply) + except ValueError: + return RedisFailure(kind="decode", message=f"session spend is not a number: {reply!r}") diff --git a/tests/unit/proxy/hooks/test_max_budget_per_session_limiter.py b/tests/unit/proxy/hooks/test_max_budget_per_session_limiter.py index 45cc7eb8922..f1c658208f5 100644 --- a/tests/unit/proxy/hooks/test_max_budget_per_session_limiter.py +++ b/tests/unit/proxy/hooks/test_max_budget_per_session_limiter.py @@ -8,20 +8,22 @@ Tests that session-scoped budget tracking works correctly: - Requests without agent_id pass through """ +import logging from unittest.mock import patch -import logging +import fakeredis import pytest from fastapi import HTTPException from litellm.caching.caching import DualCache -from litellm.caching.redis_cache import _redis_circuit_breaker_guard +from litellm.caching.redis_cache import RedisCache from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.hooks.max_budget_per_session_limiter import ( PROXY_MaxBudgetPerSessionHandler, ) from litellm.proxy.utils import InternalUsageCache from litellm.types.agents import AgentResponse +from tests._support.fake_redis import fake_redis_cache def _make_mock_agent(max_budget_per_session: float) -> AgentResponse: @@ -49,9 +51,7 @@ async def test_budget_per_session_under_budget_passes(): mock_agent = _make_mock_agent(max_budget_per_session=5.0) - with patch( - "litellm.proxy.agent_endpoints.agent_registry.global_agent_registry" - ) as mock_registry: + with patch("litellm.proxy.agent_endpoints.agent_registry.global_agent_registry") as mock_registry: mock_registry.get_agent_by_id.return_value = mock_agent result = await handler.async_pre_call_hook( @@ -79,14 +79,11 @@ async def test_budget_per_session_exceeds_budget(): ) session_id = "session-over-budget" - cache_key = handler._make_cache_key(session_id) - await handler._increment_spend(cache_key, 1.50) + await handler._increment_spend(session_id, 1.50) mock_agent = _make_mock_agent(max_budget_per_session=1.0) - with patch( - "litellm.proxy.agent_endpoints.agent_registry.global_agent_registry" - ) as mock_registry: + with patch("litellm.proxy.agent_endpoints.agent_registry.global_agent_registry") as mock_registry: mock_registry.get_agent_by_id.return_value = mock_agent with pytest.raises(HTTPException) as exc_info: @@ -115,14 +112,11 @@ async def test_budget_per_session_independent_sessions(): agent_id="agent-budget-123", ) - cache_key_a = handler._make_cache_key("session-A") - await handler._increment_spend(cache_key_a, 3.0) + await handler._increment_spend("session-A", 3.0) mock_agent = _make_mock_agent(max_budget_per_session=2.0) - with patch( - "litellm.proxy.agent_endpoints.agent_registry.global_agent_registry" - ) as mock_registry: + with patch("litellm.proxy.agent_endpoints.agent_registry.global_agent_registry") as mock_registry: mock_registry.get_agent_by_id.return_value = mock_agent # Session A should be blocked @@ -167,34 +161,81 @@ async def test_no_agent_id_passes(): assert result is None -class _OpenBreakerRedis: - def __init__(self) -> None: - from litellm.caching.redis_cache import RedisCircuitBreaker +def _redis_handler(server: fakeredis.FakeServer, namespace: str | None = None) -> tuple[RedisCache, object]: + cache = fake_redis_cache(server, namespace=namespace) + return cache, PROXY_MaxBudgetPerSessionHandler( + internal_usage_cache=InternalUsageCache(DualCache(redis_cache=cache)) + ) - self._circuit_breaker = RedisCircuitBreaker(failure_threshold=3, recovery_timeout=60) - for _ in range(3): - self._circuit_breaker.record_failure() - @_redis_circuit_breaker_guard - async def async_get_cache(self, key, **kwargs): - raise AssertionError("never reached") +async def _pre_call(handler, session_id: str, max_budget: float) -> None: + with patch("litellm.proxy.agent_endpoints.agent_registry.global_agent_registry") as mock_registry: + mock_registry.get_agent_by_id.return_value = _make_mock_agent(max_budget_per_session=max_budget) + await handler.async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth(api_key="sk-test-key-budget", agent_id="agent-budget-123"), + cache=DualCache(), + data={"metadata": {"session_id": session_id}}, + call_type="", + ) - def async_register_script(self, script): - @_redis_circuit_breaker_guard - async def refused(_self, keys, args): - raise AssertionError("never reached") - return lambda keys, args: refused(self, keys, args) +@pytest.mark.asyncio +async def test_spend_written_through_redis_is_shared_with_old_pods_and_expires(): + server = fakeredis.FakeServer() + cache, handler = _redis_handler(server, namespace="ns") + raw = fakeredis.FakeAsyncRedis(server=server) + + assert await handler._increment_spend("s1", 1.25) == 1.25 + assert await handler._increment_spend("s1", 0.5) == 1.75 + + assert await cache.async_get_cache(key="{session_budget:s1}:spend") == 1.75 + assert 3_590_000 < await raw.pttl("ns:{session_budget:s1}:spend") <= 3_600_000 + with pytest.raises(HTTPException): + await _pre_call(handler, "s1", max_budget=1.5) + + +@pytest.mark.asyncio +async def test_spend_an_old_pod_wrote_blocks_the_session(): + server = fakeredis.FakeServer() + _, handler = _redis_handler(server, namespace="ns") + await fakeredis.FakeAsyncRedis(server=server).incrbyfloat("ns:{session_budget:s1}:spend", 2.5) + + assert await handler._get_current_spend("s1") == 2.5 + await _pre_call(handler, "s1", max_budget=3.0) + with pytest.raises(HTTPException): + await _pre_call(handler, "s1", max_budget=2.5) + + +@pytest.mark.asyncio +async def test_a_spend_that_does_not_decode_falls_back_to_local_spend(): + server = fakeredis.FakeServer() + _, handler = _redis_handler(server) + await fakeredis.FakeAsyncRedis(server=server).set("{session_budget:s1}:spend", "not-a-number") + + assert await handler._get_current_spend("s1") == 0.0 + + +@pytest.mark.asyncio +async def test_an_unreachable_redis_keeps_counting_spend_locally(): + server = fakeredis.FakeServer() + _, handler = _redis_handler(server) + server.connected = False + + assert await handler._increment_spend("s1", 1.0) == 1.0 + assert await handler._increment_spend("s1", 2.0) == 3.0 + assert await handler._get_current_spend("s1") == 3.0 @pytest.mark.asyncio async def test_an_open_circuit_breaker_reads_session_spend_locally_without_a_warning(caplog): - cache = DualCache(redis_cache=_OpenBreakerRedis()) # pyright: ignore[reportArgumentType] # duck-typed Redis double - handler = PROXY_MaxBudgetPerSessionHandler(internal_usage_cache=InternalUsageCache(cache)) + server = fakeredis.FakeServer() + cache, handler = _redis_handler(server) + for _ in range(cache._circuit_breaker.failure_threshold): + cache._circuit_breaker.record_failure() caplog.clear() - with caplog.at_level(logging.DEBUG, logger="LiteLLM Proxy"): - spend = await handler._get_current_spend("{session_budget:quiet}:spend") + with caplog.at_level(logging.DEBUG): + spend = await handler._get_current_spend("quiet") assert spend == 0.0 assert [record.getMessage() for record in caplog.records if record.levelno >= logging.WARNING] == [] diff --git a/tests/unit/test_rate_limit_error_unification.py b/tests/unit/test_rate_limit_error_unification.py index ac2af7dba99..48a928886ac 100644 --- a/tests/unit/test_rate_limit_error_unification.py +++ b/tests/unit/test_rate_limit_error_unification.py @@ -18,6 +18,9 @@ the new behavior: internal modules. """ +from typing import Final + +import fakeredis import pytest from fastapi import HTTPException @@ -831,16 +834,22 @@ class TestProxyHooksActuallyRaiseProxyRateLimitError: ): """Drive `_PROXY_MaxBudgetPerSessionHandler` past its budget and assert the unified class is raised.""" - from unittest.mock import AsyncMock, MagicMock, patch + from unittest.mock import MagicMock, patch from litellm.caching.caching import DualCache from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.hooks.max_budget_per_session_limiter import ( PROXY_MaxBudgetPerSessionHandler, ) + from litellm.proxy.utils import InternalUsageCache + from tests._support.fake_redis import fake_redis_cache - internal_cache = MagicMock() - internal_cache.async_get_cache = AsyncMock(return_value=10.0) + internal_cache: Final = InternalUsageCache(DualCache(redis_cache=fake_redis_cache(fakeredis.FakeServer()))) + await internal_cache.async_set_cache( + key="{session_budget:session-over-budget}:spend", + value=10.0, + litellm_parent_otel_span=None, + ) handler = PROXY_MaxBudgetPerSessionHandler( internal_usage_cache=internal_cache, )