mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
refactor(redis-store): move session budgets behind a store
Co-authored-by: Yujong Lee <yujong@berri.ai>
This commit is contained in:
parent
8efade866f
commit
61d8a0b164
4 changed files with 163 additions and 105 deletions
|
|
@ -14,8 +14,8 @@ Works across multiple proxy instances via DualCache (in-memory + Redis).
|
|||
Follows the same pattern as max_iterations_limiter.py.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
from datetime import timedelta
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
from pydantic import ConfigDict, TypeAdapter
|
||||
|
|
@ -23,12 +23,13 @@ from pydantic import ConfigDict, TypeAdapter
|
|||
from litellm import DualCache
|
||||
from litellm._internal_context import with_service_target
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.caching.redis_cache import log_redis_failure
|
||||
from litellm.exceptions import RateLimitType
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
|
||||
from litellm.proxy.hooks.rate_limiter_utils import resolve_llm_provider_for_rate_limit
|
||||
from litellm.redis.stores.session_budget import SessionBudgetStore, TransportSessionBudgetStore, spend_key
|
||||
from litellm.redis.transport.contract import RedisFailure
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.proxy.utils import InternalUsageCache as _InternalUsageCache
|
||||
|
|
@ -38,31 +39,10 @@ else:
|
|||
InternalUsageCache = Any
|
||||
|
||||
|
||||
_CACHE_AMOUNT_ADAPTER: Final[TypeAdapter[str | bytes | int | float]] = TypeAdapter(
|
||||
str | bytes | int | float, config=ConfigDict(strict=True, hide_input_in_errors=True)
|
||||
)
|
||||
_OPTIONAL_CACHE_AMOUNT_ADAPTER: Final[TypeAdapter[str | bytes | int | float | None]] = TypeAdapter(
|
||||
str | bytes | int | float | None, config=ConfigDict(strict=True, hide_input_in_errors=True)
|
||||
)
|
||||
|
||||
|
||||
# Redis Lua script for atomic float increment with TTL.
|
||||
# INCRBYFLOAT returns the new value as a string.
|
||||
# Only sets EXPIRE on first call (when prior value was nil).
|
||||
MAX_BUDGET_SESSION_INCREMENT_SCRIPT: Final = """
|
||||
local key = KEYS[1]
|
||||
local amount = ARGV[1]
|
||||
local ttl = tonumber(ARGV[2])
|
||||
|
||||
local existed = redis.call('EXISTS', key)
|
||||
local new_val = redis.call('INCRBYFLOAT', key, amount)
|
||||
if existed == 0 then
|
||||
redis.call('EXPIRE', key, ttl)
|
||||
end
|
||||
|
||||
return new_val
|
||||
"""
|
||||
|
||||
# Default TTL for session budget counters (1 hour)
|
||||
DEFAULT_MAX_BUDGET_PER_SESSION_TTL: Final = 3600
|
||||
|
||||
|
|
@ -87,12 +67,10 @@ class _PROXY_MaxBudgetPerSessionHandler(CustomLogger):
|
|||
)
|
||||
)
|
||||
|
||||
if self.internal_usage_cache.dual_cache.redis_cache is not None:
|
||||
self.increment_script = self.internal_usage_cache.dual_cache.redis_cache.async_register_script(
|
||||
MAX_BUDGET_SESSION_INCREMENT_SCRIPT
|
||||
)
|
||||
else:
|
||||
self.increment_script = None
|
||||
redis_cache: Final = self.internal_usage_cache.dual_cache.redis_cache
|
||||
self._store: Final[SessionBudgetStore | None] = (
|
||||
TransportSessionBudgetStore(redis_cache.transport()) if redis_cache is not None else None
|
||||
)
|
||||
|
||||
@with_service_target("session_budgets")
|
||||
async def async_pre_call_hook(
|
||||
|
|
@ -114,8 +92,7 @@ class _PROXY_MaxBudgetPerSessionHandler(CustomLogger):
|
|||
return None
|
||||
|
||||
max_budget = float(max_budget)
|
||||
cache_key: Final = self._make_cache_key(session_id)
|
||||
current_spend: Final = await self._get_current_spend(cache_key)
|
||||
current_spend: Final = await self._get_current_spend(session_id)
|
||||
|
||||
verbose_proxy_logger.debug(
|
||||
"MaxBudgetPerSessionHandler: session_id=%s, spend=%.4f, max=%.2f",
|
||||
|
|
@ -172,8 +149,7 @@ class _PROXY_MaxBudgetPerSessionHandler(CustomLogger):
|
|||
if response_cost <= 0:
|
||||
return
|
||||
|
||||
cache_key: Final = self._make_cache_key(str(session_id))
|
||||
await self._increment_spend(cache_key, float(response_cost))
|
||||
await self._increment_spend(str(session_id), float(response_cost))
|
||||
|
||||
verbose_proxy_logger.debug(
|
||||
"MaxBudgetPerSessionHandler: incremented session %s spend by %.6f",
|
||||
|
|
@ -219,30 +195,19 @@ class _PROXY_MaxBudgetPerSessionHandler(CustomLogger):
|
|||
return None
|
||||
|
||||
def _make_cache_key(self, session_id: str) -> str:
|
||||
return f"{{session_budget:{session_id}}}:spend"
|
||||
return spend_key(session_id)
|
||||
|
||||
@with_service_target("session_budgets")
|
||||
async def _get_current_spend(self, cache_key: str) -> float:
|
||||
async def _get_current_spend(self, session_id: str) -> float:
|
||||
"""Read current accumulated spend for a session."""
|
||||
if self.internal_usage_cache.dual_cache.redis_cache is not None:
|
||||
try:
|
||||
redis_result: Final[str | bytes | int | float | None] = _OPTIONAL_CACHE_AMOUNT_ADAPTER.validate_python(
|
||||
await self.internal_usage_cache.dual_cache.redis_cache.async_get_cache(key=cache_key)
|
||||
)
|
||||
if redis_result is not None:
|
||||
return float(redis_result)
|
||||
return 0.0
|
||||
except Exception as e:
|
||||
log_redis_failure(
|
||||
verbose_proxy_logger,
|
||||
logging.WARNING,
|
||||
"MaxBudgetPerSessionHandler: Redis GET failed, falling back to in-memory",
|
||||
e,
|
||||
)
|
||||
if self._store is not None:
|
||||
stored: Final = await self._store.spend(session_id)
|
||||
if not isinstance(stored, RedisFailure):
|
||||
return 0.0 if stored is None else stored
|
||||
|
||||
cache_result: Final[str | bytes | int | float | None] = _OPTIONAL_CACHE_AMOUNT_ADAPTER.validate_python(
|
||||
await self.internal_usage_cache.async_get_cache(
|
||||
key=cache_key,
|
||||
key=self._make_cache_key(session_id),
|
||||
litellm_parent_otel_span=None,
|
||||
local_only=True,
|
||||
)
|
||||
|
|
@ -251,26 +216,14 @@ class _PROXY_MaxBudgetPerSessionHandler(CustomLogger):
|
|||
return float(cache_result)
|
||||
return 0.0
|
||||
|
||||
async def _increment_spend(self, cache_key: str, amount: float) -> float:
|
||||
async def _increment_spend(self, session_id: str, amount: float) -> float:
|
||||
"""Atomically increment the session spend and return the new value."""
|
||||
if self.increment_script is not None:
|
||||
try:
|
||||
script_result: Final[str | bytes | int | float] = _CACHE_AMOUNT_ADAPTER.validate_python(
|
||||
await self.increment_script(
|
||||
keys=[cache_key],
|
||||
args=[str(amount), self.ttl],
|
||||
)
|
||||
)
|
||||
return float(script_result)
|
||||
except Exception as e:
|
||||
log_redis_failure(
|
||||
verbose_proxy_logger,
|
||||
logging.WARNING,
|
||||
"MaxBudgetPerSessionHandler: Redis INCRBYFLOAT failed, falling back to in-memory",
|
||||
e,
|
||||
)
|
||||
if self._store is not None:
|
||||
added: Final = await self._store.add_spend(session_id, amount, ttl=timedelta(seconds=self.ttl))
|
||||
if not isinstance(added, RedisFailure):
|
||||
return added
|
||||
|
||||
return await self._in_memory_increment_spend(cache_key, amount)
|
||||
return await self._in_memory_increment_spend(self._make_cache_key(session_id), amount)
|
||||
|
||||
async def _in_memory_increment_spend(self, cache_key: str, amount: float) -> float:
|
||||
current: Final[str | bytes | int | float | None] = _OPTIONAL_CACHE_AMOUNT_ADAPTER.validate_python(
|
||||
|
|
|
|||
55
litellm/redis/stores/session_budget.py
Normal file
55
litellm/redis/stores/session_budget.py
Normal file
|
|
@ -0,0 +1,55 @@
|
|||
from dataclasses import dataclass
|
||||
from datetime import timedelta
|
||||
from typing import Final, Protocol
|
||||
|
||||
from litellm.redis.transport.contract import EvalReply, RedisFailure, RedisTransport
|
||||
|
||||
_ADD_SPEND: Final = """
|
||||
local key = KEYS[1]
|
||||
local amount = ARGV[1]
|
||||
local ttl = tonumber(ARGV[2])
|
||||
|
||||
local existed = redis.call('EXISTS', key)
|
||||
local new_val = redis.call('INCRBYFLOAT', key, amount)
|
||||
if existed == 0 then
|
||||
redis.call('EXPIRE', key, ttl)
|
||||
end
|
||||
|
||||
return new_val
|
||||
"""
|
||||
|
||||
|
||||
def spend_key(session_id: str) -> str:
|
||||
return f"{{session_budget:{session_id}}}:spend"
|
||||
|
||||
|
||||
class SessionBudgetStore(Protocol):
|
||||
async def add_spend(self, session_id: str, amount: float, *, ttl: timedelta) -> float | RedisFailure: ...
|
||||
|
||||
async def spend(self, session_id: str) -> float | None | RedisFailure: ...
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class TransportSessionBudgetStore:
|
||||
transport: RedisTransport
|
||||
|
||||
async def add_spend(self, session_id: str, amount: float, *, ttl: timedelta) -> float | RedisFailure:
|
||||
reply: Final = await self.transport.eval(
|
||||
_ADD_SPEND, (spend_key(session_id),), (str(amount), ttl // timedelta(seconds=1))
|
||||
)
|
||||
return reply if isinstance(reply, RedisFailure) else _amount(reply)
|
||||
|
||||
async def spend(self, session_id: str) -> float | None | RedisFailure:
|
||||
raw: Final = await self.transport.get(spend_key(session_id))
|
||||
if raw is None or isinstance(raw, RedisFailure):
|
||||
return raw
|
||||
return _amount(raw)
|
||||
|
||||
|
||||
def _amount(reply: EvalReply) -> float | RedisFailure:
|
||||
if not isinstance(reply, (bytes, int)):
|
||||
return RedisFailure(kind="decode", message=f"session spend is not a number: {reply!r}")
|
||||
try:
|
||||
return float(reply)
|
||||
except ValueError:
|
||||
return RedisFailure(kind="decode", message=f"session spend is not a number: {reply!r}")
|
||||
|
|
@ -8,20 +8,22 @@ Tests that session-scoped budget tracking works correctly:
|
|||
- Requests without agent_id pass through
|
||||
"""
|
||||
|
||||
import logging
|
||||
from unittest.mock import patch
|
||||
|
||||
import logging
|
||||
import fakeredis
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
|
||||
from litellm.caching.caching import DualCache
|
||||
from litellm.caching.redis_cache import _redis_circuit_breaker_guard
|
||||
from litellm.caching.redis_cache import RedisCache
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.hooks.max_budget_per_session_limiter import (
|
||||
PROXY_MaxBudgetPerSessionHandler,
|
||||
)
|
||||
from litellm.proxy.utils import InternalUsageCache
|
||||
from litellm.types.agents import AgentResponse
|
||||
from tests._support.fake_redis import fake_redis_cache
|
||||
|
||||
|
||||
def _make_mock_agent(max_budget_per_session: float) -> AgentResponse:
|
||||
|
|
@ -49,9 +51,7 @@ async def test_budget_per_session_under_budget_passes():
|
|||
|
||||
mock_agent = _make_mock_agent(max_budget_per_session=5.0)
|
||||
|
||||
with patch(
|
||||
"litellm.proxy.agent_endpoints.agent_registry.global_agent_registry"
|
||||
) as mock_registry:
|
||||
with patch("litellm.proxy.agent_endpoints.agent_registry.global_agent_registry") as mock_registry:
|
||||
mock_registry.get_agent_by_id.return_value = mock_agent
|
||||
|
||||
result = await handler.async_pre_call_hook(
|
||||
|
|
@ -79,14 +79,11 @@ async def test_budget_per_session_exceeds_budget():
|
|||
)
|
||||
|
||||
session_id = "session-over-budget"
|
||||
cache_key = handler._make_cache_key(session_id)
|
||||
await handler._increment_spend(cache_key, 1.50)
|
||||
await handler._increment_spend(session_id, 1.50)
|
||||
|
||||
mock_agent = _make_mock_agent(max_budget_per_session=1.0)
|
||||
|
||||
with patch(
|
||||
"litellm.proxy.agent_endpoints.agent_registry.global_agent_registry"
|
||||
) as mock_registry:
|
||||
with patch("litellm.proxy.agent_endpoints.agent_registry.global_agent_registry") as mock_registry:
|
||||
mock_registry.get_agent_by_id.return_value = mock_agent
|
||||
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
|
|
@ -115,14 +112,11 @@ async def test_budget_per_session_independent_sessions():
|
|||
agent_id="agent-budget-123",
|
||||
)
|
||||
|
||||
cache_key_a = handler._make_cache_key("session-A")
|
||||
await handler._increment_spend(cache_key_a, 3.0)
|
||||
await handler._increment_spend("session-A", 3.0)
|
||||
|
||||
mock_agent = _make_mock_agent(max_budget_per_session=2.0)
|
||||
|
||||
with patch(
|
||||
"litellm.proxy.agent_endpoints.agent_registry.global_agent_registry"
|
||||
) as mock_registry:
|
||||
with patch("litellm.proxy.agent_endpoints.agent_registry.global_agent_registry") as mock_registry:
|
||||
mock_registry.get_agent_by_id.return_value = mock_agent
|
||||
|
||||
# Session A should be blocked
|
||||
|
|
@ -167,34 +161,81 @@ async def test_no_agent_id_passes():
|
|||
assert result is None
|
||||
|
||||
|
||||
class _OpenBreakerRedis:
|
||||
def __init__(self) -> None:
|
||||
from litellm.caching.redis_cache import RedisCircuitBreaker
|
||||
def _redis_handler(server: fakeredis.FakeServer, namespace: str | None = None) -> tuple[RedisCache, object]:
|
||||
cache = fake_redis_cache(server, namespace=namespace)
|
||||
return cache, PROXY_MaxBudgetPerSessionHandler(
|
||||
internal_usage_cache=InternalUsageCache(DualCache(redis_cache=cache))
|
||||
)
|
||||
|
||||
self._circuit_breaker = RedisCircuitBreaker(failure_threshold=3, recovery_timeout=60)
|
||||
for _ in range(3):
|
||||
self._circuit_breaker.record_failure()
|
||||
|
||||
@_redis_circuit_breaker_guard
|
||||
async def async_get_cache(self, key, **kwargs):
|
||||
raise AssertionError("never reached")
|
||||
async def _pre_call(handler, session_id: str, max_budget: float) -> None:
|
||||
with patch("litellm.proxy.agent_endpoints.agent_registry.global_agent_registry") as mock_registry:
|
||||
mock_registry.get_agent_by_id.return_value = _make_mock_agent(max_budget_per_session=max_budget)
|
||||
await handler.async_pre_call_hook(
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="sk-test-key-budget", agent_id="agent-budget-123"),
|
||||
cache=DualCache(),
|
||||
data={"metadata": {"session_id": session_id}},
|
||||
call_type="",
|
||||
)
|
||||
|
||||
def async_register_script(self, script):
|
||||
@_redis_circuit_breaker_guard
|
||||
async def refused(_self, keys, args):
|
||||
raise AssertionError("never reached")
|
||||
|
||||
return lambda keys, args: refused(self, keys, args)
|
||||
@pytest.mark.asyncio
|
||||
async def test_spend_written_through_redis_is_shared_with_old_pods_and_expires():
|
||||
server = fakeredis.FakeServer()
|
||||
cache, handler = _redis_handler(server, namespace="ns")
|
||||
raw = fakeredis.FakeAsyncRedis(server=server)
|
||||
|
||||
assert await handler._increment_spend("s1", 1.25) == 1.25
|
||||
assert await handler._increment_spend("s1", 0.5) == 1.75
|
||||
|
||||
assert await cache.async_get_cache(key="{session_budget:s1}:spend") == 1.75
|
||||
assert 3_590_000 < await raw.pttl("ns:{session_budget:s1}:spend") <= 3_600_000
|
||||
with pytest.raises(HTTPException):
|
||||
await _pre_call(handler, "s1", max_budget=1.5)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_spend_an_old_pod_wrote_blocks_the_session():
|
||||
server = fakeredis.FakeServer()
|
||||
_, handler = _redis_handler(server, namespace="ns")
|
||||
await fakeredis.FakeAsyncRedis(server=server).incrbyfloat("ns:{session_budget:s1}:spend", 2.5)
|
||||
|
||||
assert await handler._get_current_spend("s1") == 2.5
|
||||
await _pre_call(handler, "s1", max_budget=3.0)
|
||||
with pytest.raises(HTTPException):
|
||||
await _pre_call(handler, "s1", max_budget=2.5)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_spend_that_does_not_decode_falls_back_to_local_spend():
|
||||
server = fakeredis.FakeServer()
|
||||
_, handler = _redis_handler(server)
|
||||
await fakeredis.FakeAsyncRedis(server=server).set("{session_budget:s1}:spend", "not-a-number")
|
||||
|
||||
assert await handler._get_current_spend("s1") == 0.0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_unreachable_redis_keeps_counting_spend_locally():
|
||||
server = fakeredis.FakeServer()
|
||||
_, handler = _redis_handler(server)
|
||||
server.connected = False
|
||||
|
||||
assert await handler._increment_spend("s1", 1.0) == 1.0
|
||||
assert await handler._increment_spend("s1", 2.0) == 3.0
|
||||
assert await handler._get_current_spend("s1") == 3.0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_open_circuit_breaker_reads_session_spend_locally_without_a_warning(caplog):
|
||||
cache = DualCache(redis_cache=_OpenBreakerRedis()) # pyright: ignore[reportArgumentType] # duck-typed Redis double
|
||||
handler = PROXY_MaxBudgetPerSessionHandler(internal_usage_cache=InternalUsageCache(cache))
|
||||
server = fakeredis.FakeServer()
|
||||
cache, handler = _redis_handler(server)
|
||||
for _ in range(cache._circuit_breaker.failure_threshold):
|
||||
cache._circuit_breaker.record_failure()
|
||||
caplog.clear()
|
||||
|
||||
with caplog.at_level(logging.DEBUG, logger="LiteLLM Proxy"):
|
||||
spend = await handler._get_current_spend("{session_budget:quiet}:spend")
|
||||
with caplog.at_level(logging.DEBUG):
|
||||
spend = await handler._get_current_spend("quiet")
|
||||
|
||||
assert spend == 0.0
|
||||
assert [record.getMessage() for record in caplog.records if record.levelno >= logging.WARNING] == []
|
||||
|
|
|
|||
|
|
@ -18,6 +18,9 @@ the new behavior:
|
|||
internal modules.
|
||||
"""
|
||||
|
||||
from typing import Final
|
||||
|
||||
import fakeredis
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
|
||||
|
|
@ -831,16 +834,22 @@ class TestProxyHooksActuallyRaiseProxyRateLimitError:
|
|||
):
|
||||
"""Drive `_PROXY_MaxBudgetPerSessionHandler` past its budget and
|
||||
assert the unified class is raised."""
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from litellm.caching.caching import DualCache
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.hooks.max_budget_per_session_limiter import (
|
||||
PROXY_MaxBudgetPerSessionHandler,
|
||||
)
|
||||
from litellm.proxy.utils import InternalUsageCache
|
||||
from tests._support.fake_redis import fake_redis_cache
|
||||
|
||||
internal_cache = MagicMock()
|
||||
internal_cache.async_get_cache = AsyncMock(return_value=10.0)
|
||||
internal_cache: Final = InternalUsageCache(DualCache(redis_cache=fake_redis_cache(fakeredis.FakeServer())))
|
||||
await internal_cache.async_set_cache(
|
||||
key="{session_budget:session-over-budget}:spend",
|
||||
value=10.0,
|
||||
litellm_parent_otel_span=None,
|
||||
)
|
||||
handler = PROXY_MaxBudgetPerSessionHandler(
|
||||
internal_usage_cache=internal_cache,
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue