This commit is contained in:
Alisina Karimi 2026-10-27 21:10:14 +00:00 • committed by GitHub
commit c1d539bd0d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 1583 additions and 183 deletions

View file

@ -1,11 +1,12 @@
"""
Per-Session Budget Limiter for LiteLLM Proxy.
Enforces a dollar-amount cap per session (identified by `session_id` /
`x-litellm-trace-id`). After each successful LLM call the response cost is
accumulated against the session. When the accumulated spend exceeds
`max_budget_per_session` (configured in agent litellm_params), subsequent
requests for that session receive a 429.
Stops new calls once recorded spend for an agent and session reaches
`max_budget_per_session` (configured in agent litellm_params). Sessions are
identified by `session_id` / `x-litellm-trace-id`. Costs are recorded after
successful responses, with no reservation before forwarding. Calls admitted
before those responses arrive can finish above the budget. Subsequent requests
receive a 429 once recorded spend is at or above the threshold.
Note: trace-id enforcement (require_trace_id_on_calls_by_agent) is handled
separately in auth_checks.py at the agent level, not in this hook.
@ -14,9 +15,18 @@ Works across multiple proxy instances via DualCache (in-memory + Redis).
Follows the same pattern as max_iterations_limiter.py.
"""
import asyncio
import json
import logging
import math
import os
from typing import TYPE_CHECKING, Any, Final
from collections.abc import Awaitable, Callable, Mapping
from typing import (
TYPE_CHECKING,
Any,
Final,
cast, # noqa: TID251 # Redis/cache APIs expose dynamic values narrowed at this boundary
)
from litellm import DualCache
from litellm._internal_context import with_service_target
@ -36,21 +46,126 @@ else:
InternalUsageCache = Any
# Redis Lua script for atomic float increment with TTL.
# INCRBYFLOAT returns the new value as a string.
# Only sets EXPIRE on first call (when prior value was nil).
# Redis Lua scripts keep the legacy aggregate and new per-agent counters in
# sync. All keys use the legacy session hash tag so this also works on Redis
# Cluster. Old proxy instances continue to update the aggregate key.
MAX_BUDGET_SESSION_INCREMENT_SCRIPT: Final = """
local key = KEYS[1]
local amount = ARGV[1]
local legacy_key = KEYS[1]
local agent_scope_key = KEYS[2]
local amount = tonumber(ARGV[1])
local ttl = tonumber(ARGV[2])
local existed = redis.call('EXISTS', key)
local new_val = redis.call('INCRBYFLOAT', key, amount)
if existed == 0 then
redis.call('EXPIRE', key, ttl)
local agent_field = ARGV[3]
local function parse_spend(raw)
if raw == false then return 0 end
if string.match(raw, '^%s') or string.match(raw, '%s$') or string.find(raw, '%z') then
return nil
end
return tonumber(raw)
end
if amount == nil or amount <= 0 or amount ~= amount or math.abs(amount) == math.huge then
return redis.error_reply('agent session spend increment is invalid')
end
return new_val
local legacy_raw = redis.call('GET', legacy_key)
local previous_legacy = parse_spend(legacy_raw)
if previous_legacy == nil or previous_legacy < 0 or previous_legacy ~= previous_legacy
or math.abs(previous_legacy) == math.huge then
return redis.error_reply('legacy session spend is invalid')
end
if redis.call('EXISTS', agent_scope_key) == 0 then
if redis.call('EXISTS', legacy_key) == 0 then
redis.call('SET', legacy_key, '0')
redis.call('PEXPIRE', legacy_key, ttl * 1000)
end
local legacy_ttl = redis.call('PTTL', legacy_key)
if legacy_ttl == -2 then
return redis.error_reply('legacy session spend expired during migration')
end
if legacy_ttl >= 0 and legacy_ttl <= 1 then
return redis.error_reply('legacy session spend is expiring before agent scope')
end
redis.call('HSET', agent_scope_key, '__total_new', '0')
if legacy_ttl >= 0 then
redis.call('PEXPIRE', agent_scope_key, legacy_ttl - 1)
end
end
if redis.call('EXISTS', legacy_key) == 0 then
return redis.error_reply('legacy session spend expired before agent scope')
end
local total_new_raw = redis.call('HGET', agent_scope_key, '__total_new')
if total_new_raw == false then
return redis.error_reply('agent session scope is missing its migration total')
end
local total_new_value = parse_spend(total_new_raw)
if total_new_value == nil or total_new_value < 0 or total_new_value ~= total_new_value
or math.abs(total_new_value) == math.huge then
return redis.error_reply('agent session migration total is not numeric')
end
local agent_raw = redis.call('HGET', agent_scope_key, agent_field)
local agent_value = parse_spend(agent_raw)
if agent_value == nil or agent_value < 0 or agent_value ~= agent_value or math.abs(agent_value) == math.huge then
return redis.error_reply('agent session counter is not numeric')
end
if math.abs(previous_legacy + amount) == math.huge or math.abs(total_new_value + amount) == math.huge
or math.abs(agent_value + amount) == math.huge then
return redis.error_reply('agent session spend increment is out of range')
end
local legacy_value = tonumber(redis.call('INCRBYFLOAT', legacy_key, amount))
local next_total_new = tonumber(redis.call('HINCRBYFLOAT', agent_scope_key, '__total_new', amount))
local next_agent_value = tonumber(redis.call('HINCRBYFLOAT', agent_scope_key, agent_field, amount))
return tostring(math.max(legacy_value - next_total_new, 0) + next_agent_value)
"""
MAX_BUDGET_SESSION_GET_AGENT_SPEND_SCRIPT: Final = """
local legacy_key = KEYS[1]
local agent_scope_key = KEYS[2]
local function parse_spend(raw)
if raw == false then return 0 end
if string.match(raw, '^%s') or string.match(raw, '%s$') or string.find(raw, '%z') then
return nil
end
return tonumber(raw)
end
local legacy_raw = redis.call('GET', legacy_key)
local legacy_value = parse_spend(legacy_raw)
if legacy_value == nil or legacy_value < 0 or legacy_value ~= legacy_value or math.abs(legacy_value) == math.huge then
return redis.error_reply('legacy session spend is invalid')
end
if redis.call('EXISTS', agent_scope_key) == 0 then
return tostring(legacy_value)
end
if redis.call('EXISTS', legacy_key) == 0 then
return redis.error_reply('legacy session spend expired before agent scope')
end
local total_new_raw = redis.call('HGET', agent_scope_key, '__total_new')
if total_new_raw == false then
return redis.error_reply('agent session scope is missing its migration total')
end
local total_new_value = parse_spend(total_new_raw)
if total_new_value == nil or total_new_value < 0 or total_new_value ~= total_new_value
or math.abs(total_new_value) == math.huge then
return redis.error_reply('agent session migration total is not numeric')
end
local agent_raw = redis.call('HGET', agent_scope_key, ARGV[1])
local agent_value = 0
if agent_raw ~= false then
agent_value = parse_spend(agent_raw)
if agent_value == nil or agent_value < 0 or agent_value ~= agent_value or math.abs(agent_value) == math.huge then
return redis.error_reply('agent session counter is not numeric')
end
end
return tostring(math.max(legacy_value - total_new_value, 0) + agent_value)
"""
# Default TTL for session budget counters (1 hour)
@ -59,17 +174,25 @@ DEFAULT_MAX_BUDGET_PER_SESSION_TTL: Final = 3600
class _PROXY_MaxBudgetPerSessionHandler(CustomLogger):
"""
Pre-call hook that enforces max_budget_per_session.
Pre-call hook that checks the max_budget_per_session stop threshold.
Configuration (set in agent litellm_params):
- max_budget_per_session: dollar cap per session_id
- max_budget_per_session: recorded-spend threshold per agent and session_id
Cache key pattern:
{session_budget:<session_id>}:spend
Costs are recorded after successful responses. Concurrent calls can pass
the check before their costs are recorded and finish above the threshold.
Cache keys:
{session_budget:<session_id>}:spend (legacy aggregate)
{session_budget:<session_id>}:agent-scope (per-agent migration ledger)
"""
def __init__(self, internal_usage_cache: InternalUsageCache):
self.internal_usage_cache = internal_usage_cache
self._local_lock = asyncio.Lock()
self.increment_script: Callable[..., Awaitable[object]] | None = None
self.get_agent_spend_script: Callable[..., Awaitable[object]] | None = None
self._registered_redis_cache: object | None = None
self.ttl = int(
os.getenv(
"LITELLM_MAX_BUDGET_PER_SESSION_TTL",
@ -77,12 +200,27 @@ class _PROXY_MaxBudgetPerSessionHandler(CustomLogger):
)
)
if self.internal_usage_cache.dual_cache.redis_cache is not None:
self.increment_script = self.internal_usage_cache.dual_cache.redis_cache.async_register_script(
MAX_BUDGET_SESSION_INCREMENT_SCRIPT
)
else:
self._ensure_redis_scripts()
def _ensure_redis_scripts(self) -> None:
redis_cache: Final = self.internal_usage_cache.dual_cache.redis_cache
if redis_cache is None:
self.increment_script = None
self.get_agent_spend_script = None
self._registered_redis_cache = None
return
if redis_cache is self._registered_redis_cache:
return
self.increment_script = cast( # cast-ok: Redis registration returns the callable invoked below
Callable[..., Awaitable[object]],
redis_cache.async_register_script(MAX_BUDGET_SESSION_INCREMENT_SCRIPT),
)
self.get_agent_spend_script = cast( # cast-ok: Redis registration returns the callable invoked below
Callable[..., Awaitable[object]],
redis_cache.async_register_script(MAX_BUDGET_SESSION_GET_AGENT_SPEND_SCRIPT),
)
self._registered_redis_cache = redis_cache
@with_service_target("session_budgets")
async def async_pre_call_hook(
@ -93,19 +231,19 @@ class _PROXY_MaxBudgetPerSessionHandler(CustomLogger):
call_type: str,
) -> Exception | str | dict | None:
"""
Before each LLM call, check if max_budget_per_session is set and
whether accumulated spend exceeds the budget (429 if so).
Reject at or above the recorded-spend threshold without reserving cost
for calls that pass the check.
"""
max_budget = self._get_max_budget_per_session(user_api_key_dict)
session_id: Final = self._get_session_id(data)
agent_id: Final = user_api_key_dict.agent_id
if max_budget is None or session_id is None:
if max_budget is None or session_id is None or agent_id is None:
return None
max_budget = float(max_budget)
cache_key: Final = self._make_cache_key(session_id)
current_spend: Final = await self._get_current_spend(cache_key)
current_spend: Final = await self._get_agent_spend(session_id, agent_id)
verbose_proxy_logger.debug(
"MaxBudgetPerSessionHandler: session_id=%s, spend=%.4f, max=%.2f",
@ -132,7 +270,9 @@ class _PROXY_MaxBudgetPerSessionHandler(CustomLogger):
@with_service_target("session_budgets")
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time) -> None:
"""
After a successful LLM call, increment the session spend by the response cost.
Record every successful agent call so limits added later still see the
session's accumulated spend. A configured threshold stops later calls
after recorded spend reaches it.
"""
try:
litellm_params: Final = kwargs.get("litellm_params") or {}
@ -141,7 +281,7 @@ class _PROXY_MaxBudgetPerSessionHandler(CustomLogger):
if session_id is None:
return
agent_id: Final = metadata.get("agent_id")
agent_id: Final = metadata.get("billing_agent_id") or metadata.get("agent_id")
if agent_id is None:
return
@ -153,17 +293,11 @@ class _PROXY_MaxBudgetPerSessionHandler(CustomLogger):
if agent is None:
return
agent_litellm_params: Final = agent.litellm_params or {}
max_budget: Final = agent_litellm_params.get("max_budget_per_session")
if max_budget is None:
return
response_cost: Final = kwargs.get("response_cost") or 0.0
if response_cost <= 0:
return
cache_key: Final = self._make_cache_key(str(session_id))
await self._increment_spend(cache_key, float(response_cost))
await self._increment_agent_spend(str(session_id), agent.agent_id, float(response_cost))
verbose_proxy_logger.debug(
"MaxBudgetPerSessionHandler: incremented session %s spend by %.6f",
@ -175,6 +309,7 @@ class _PROXY_MaxBudgetPerSessionHandler(CustomLogger):
"MaxBudgetPerSessionHandler: error in async_log_success_event: %s",
str(e),
)
raise
def _get_session_id(self, data: dict) -> str | None:
"""Extract session_id from request metadata."""
@ -208,69 +343,161 @@ class _PROXY_MaxBudgetPerSessionHandler(CustomLogger):
return float(max_budget)
return None
def _make_cache_key(self, session_id: str) -> str:
def _make_agent_scope_cache_key(self, session_id: str) -> str:
return f"{{session_budget:{session_id}}}:agent-scope"
def _make_agent_scope_field(self, agent_id: str) -> str:
from litellm.proxy.agent_endpoints.agent_registry import global_agent_registry
stable_agent_id: Final = json.dumps(global_agent_registry.stable_agent_id(agent_id), separators=(",", ":"))
return f"agent:{stable_agent_id}"
def _make_legacy_cache_key(self, session_id: str) -> str:
return f"{{session_budget:{session_id}}}:spend"
async def _get_local_scope(self, cache_key: str) -> Mapping[str, object] | None:
result: Final[object | None] = cast( # cast-ok: cache API returns Any; validate value before use
object | None,
await self.internal_usage_cache.async_get_cache(
key=cache_key,
litellm_parent_otel_span=None,
local_only=True,
),
)
if result is None:
return None
if isinstance(result, dict) and all(
isinstance(key, str)
for key in cast(Mapping[object, object], result) # cast-ok: dict shape is checked and keys remain opaque
):
return cast(Mapping[str, object], result) # cast-ok: keys are checked and values remain opaque
raise RuntimeError("Agent session scope cache has an invalid value")
@staticmethod
def _parse_spend(value: object) -> float:
if isinstance(value, bool) or not isinstance(value, (int, float, str, bytes)):
raise TypeError("Agent session spend counter is invalid")
if isinstance(value, (str, bytes)) and value != value.strip():
raise RuntimeError("Agent session spend counter is invalid")
spend: Final = float(value)
if not math.isfinite(spend) or spend < 0:
raise RuntimeError("Agent session spend counter is invalid")
return spend
@staticmethod
def _get_scope_spend(scope: Mapping[str, object], field: str) -> float:
return _PROXY_MaxBudgetPerSessionHandler._parse_spend(scope.get(field, 0.0))
@with_service_target("session_budgets")
async def _get_current_spend(self, cache_key: str) -> float:
"""Read current accumulated spend for a session."""
if self.internal_usage_cache.dual_cache.redis_cache is not None:
async def _get_agent_spend(self, session_id: str, agent_id: str) -> float:
legacy_key: Final = self._make_legacy_cache_key(session_id)
agent_scope_key: Final = self._make_agent_scope_cache_key(session_id)
agent_field: Final = self._make_agent_scope_field(agent_id)
self._ensure_redis_scripts()
if self.get_agent_spend_script is not None:
try:
result = await self.internal_usage_cache.dual_cache.redis_cache.async_get_cache(key=cache_key)
if result is not None:
return float(result)
return 0.0
result: Final[object] = await self.get_agent_spend_script(
keys=[legacy_key, agent_scope_key],
args=[agent_field],
)
if isinstance(result, (int, float, str, bytes)):
return self._parse_spend(result)
raise TypeError(f"Unexpected Redis spend result: {type(result).__name__}")
except Exception as e:
log_redis_failure(
verbose_proxy_logger,
logging.WARNING,
"MaxBudgetPerSessionHandler: Redis GET failed, falling back to in-memory",
"MaxBudgetPerSessionHandler: Redis agent spend read failed",
e,
)
raise
result = await self.internal_usage_cache.async_get_cache(
key=cache_key,
litellm_parent_otel_span=None,
local_only=True,
legacy_value: Final = await self._get_local_spend(legacy_key)
agent_scope: Final = await self._get_local_scope(agent_scope_key)
if legacy_value is None:
if agent_scope is not None:
raise RuntimeError("Legacy session spend expired before agent scope")
return 0.0
if agent_scope is None:
return float(legacy_value)
raw_total_new: Final = agent_scope.get("__total_new")
if raw_total_new is None:
raise RuntimeError("Agent session scope is missing its migration total")
total_new_value: Final = self._parse_spend(raw_total_new)
agent_value: Final = self._get_scope_spend(agent_scope, agent_field)
return max(float(legacy_value) - total_new_value, 0.0) + agent_value
async def _get_local_spend(self, cache_key: str) -> float | None:
result: Final[object | None] = cast( # cast-ok: cache API returns Any; validate value before use
object | None,
await self.internal_usage_cache.async_get_cache(
key=cache_key,
litellm_parent_otel_span=None,
local_only=True,
),
)
if result is not None:
return float(result)
return 0.0
return None if result is None else self._parse_spend(result)
async def _increment_spend(self, cache_key: str, amount: float) -> float:
"""Atomically increment the session spend and return the new value."""
async def _increment_agent_spend(self, session_id: str, agent_id: str, amount: float) -> float:
validated_amount: Final = self._parse_spend(amount)
if validated_amount == 0:
raise RuntimeError("Agent session spend increment must be positive")
legacy_key: Final = self._make_legacy_cache_key(session_id)
agent_scope_key: Final = self._make_agent_scope_cache_key(session_id)
agent_field: Final = self._make_agent_scope_field(agent_id)
self._ensure_redis_scripts()
if self.increment_script is not None:
try:
result: Final = await self.increment_script(
keys=[cache_key],
args=[str(amount), self.ttl],
result: Final[object] = await self.increment_script(
keys=[legacy_key, agent_scope_key],
args=[str(validated_amount), self.ttl, agent_field],
)
return float(result)
if isinstance(result, (int, float, str, bytes)):
return self._parse_spend(result)
raise TypeError(f"Unexpected Redis spend result: {type(result).__name__}")
except Exception as e:
log_redis_failure(
verbose_proxy_logger,
logging.WARNING,
"MaxBudgetPerSessionHandler: Redis INCRBYFLOAT failed, falling back to in-memory",
"MaxBudgetPerSessionHandler: Redis migration increment failed; refusing an unsafe retry",
e,
)
raise
return await self._in_memory_increment_spend(cache_key, amount)
async def _in_memory_increment_spend(self, cache_key: str, amount: float) -> float:
current: Final = await self.internal_usage_cache.async_get_cache(
key=cache_key,
litellm_parent_otel_span=None,
local_only=True,
)
new_value: Final = (float(current) if current is not None else 0.0) + amount
await self.internal_usage_cache.async_set_cache(
key=cache_key,
value=new_value,
ttl=self.ttl,
litellm_parent_otel_span=None,
local_only=True,
)
return new_value
async with self._local_lock:
cached_legacy: Final = await self._get_local_spend(legacy_key)
agent_scope: Final = await self._get_local_scope(agent_scope_key)
if cached_legacy is None and agent_scope is not None:
raise RuntimeError("Legacy session spend expired before agent scope")
legacy_value: Final = cached_legacy if cached_legacy is not None else 0.0
raw_total_new: Final = agent_scope.get("__total_new") if agent_scope is not None else 0.0
if raw_total_new is None:
raise RuntimeError("Agent session scope is missing its migration total")
total_new_value: Final = self._parse_spend(raw_total_new)
agent_value: Final = self._get_scope_spend(agent_scope, agent_field) if agent_scope is not None else 0.0
new_legacy: Final = self._parse_spend(float(legacy_value) + validated_amount)
new_total_new: Final = self._parse_spend(float(total_new_value) + validated_amount)
new_agent: Final = self._parse_spend(float(agent_value) + validated_amount)
await self.internal_usage_cache.async_set_cache(
key=legacy_key,
value=new_legacy,
ttl=self.ttl,
litellm_parent_otel_span=None,
local_only=True,
)
await self.internal_usage_cache.async_set_cache(
key=agent_scope_key,
value={
**(agent_scope or {}),
"__total_new": new_total_new,
agent_field: new_agent,
},
# Keep this map on a shorter rolling TTL than the aggregate fallback.
ttl=max(self.ttl - 1, 0),
litellm_parent_otel_span=None,
local_only=True,
)
return max(new_legacy - new_total_new, 0.0) + new_agent
PROXY_MaxBudgetPerSessionHandler: Final = _PROXY_MaxBudgetPerSessionHandler

View file

@ -1,7 +1,7 @@
"""
Max Iterations Limiter for LiteLLM Proxy.
Enforces a per-session cap on the number of LLM calls an agentic loop can make.
Enforces a per-agent, per-session cap on the number of LLM calls an agentic loop can make.
Callers send a `session_id` with each request (via `x-litellm-session-id` header
or `metadata.session_id`), and this hook counts calls per session. When the count
exceeds `max_iterations` (configured in agent litellm_params or key metadata), returns 429.
@ -10,8 +10,17 @@ Works across multiple proxy instances via DualCache (in-memory + Redis).
Follows the same pattern as parallel_request_limiter_v3.py.
"""
import asyncio
import json
import math
import os
from typing import TYPE_CHECKING, Any, Final
from collections.abc import Awaitable, Callable, Mapping
from typing import (
TYPE_CHECKING,
Any,
Final,
cast, # noqa: TID251 # Dynamic Redis and cache boundary values are validated before use
)
from litellm import DualCache
from litellm._internal_context import with_service_target
@ -30,19 +39,79 @@ else:
InternalUsageCache = Any
# Redis Lua script for atomic increment with TTL.
# Returns the new count after increment.
# Only sets EXPIRE on first increment (when count becomes 1).
# Redis Lua scripts keep the legacy aggregate and new per-agent counters in
# sync. All keys use the legacy session hash tag so this also works on Redis
# Cluster. Old proxy instances continue to update the aggregate key.
MAX_ITERATIONS_INCREMENT_SCRIPT: Final = """
local key = KEYS[1]
local ttl = tonumber(ARGV[1])
local current = redis.call('INCR', key)
if current == 1 then
redis.call('EXPIRE', key, ttl)
local legacy_key = KEYS[1]
if #KEYS == 1 then
local current = redis.call('INCR', legacy_key)
if current == 1 then
redis.call('EXPIRE', legacy_key, tonumber(ARGV[1]))
end
return current
end
return current
local agent_scope_key = KEYS[2]
local ttl = tonumber(ARGV[1])
local agent_field = ARGV[2]
local function is_count(raw)
return raw == '0' or string.match(raw, '^[1-9]%d*$') ~= nil
end
local legacy_raw = redis.call('GET', legacy_key)
if legacy_raw ~= false and not is_count(legacy_raw) then
return redis.error_reply('legacy session count is not a non-negative integer')
end
if redis.call('EXISTS', agent_scope_key) == 0 then
if redis.call('EXISTS', legacy_key) == 0 then
redis.call('SET', legacy_key, '0')
redis.call('PEXPIRE', legacy_key, ttl * 1000)
end
local legacy_ttl = redis.call('PTTL', legacy_key)
if legacy_ttl == -2 then
return redis.error_reply('legacy session count expired during migration')
end
if legacy_ttl >= 0 and legacy_ttl <= 1 then
return redis.error_reply('legacy session count is expiring before agent scope')
end
redis.call('HSET', agent_scope_key, '__total_new', '0')
if legacy_ttl >= 0 then
redis.call('PEXPIRE', agent_scope_key, legacy_ttl - 1)
end
end
if redis.call('EXISTS', legacy_key) == 0 then
return redis.error_reply('legacy session count expired before agent scope')
end
local total_new_raw = redis.call('HGET', agent_scope_key, '__total_new')
if total_new_raw == false then
return redis.error_reply('agent session scope is missing its migration total')
end
if not is_count(total_new_raw) then
return redis.error_reply('agent session migration total is not an integer')
end
local validated_total_new = tonumber(total_new_raw)
if validated_total_new == nil or validated_total_new >= 9223372036854774784 then
return redis.error_reply('agent session migration total is out of range')
end
local agent_raw = redis.call('HGET', agent_scope_key, agent_field)
if agent_raw ~= false and not is_count(agent_raw) then
return redis.error_reply('agent session counter is not an integer')
end
local validated_agent_value = tonumber(agent_raw or '0')
if validated_agent_value >= 9223372036854774784 then
return redis.error_reply('agent session counter is out of range')
end
local legacy_value = redis.call('INCR', legacy_key)
local total_new_value = redis.call('HINCRBY', agent_scope_key, '__total_new', 1)
local agent_value = redis.call('HINCRBY', agent_scope_key, agent_field, 1)
return math.max(legacy_value - total_new_value, 0) + agent_value
"""
# Default TTL for session iteration counters (1 hour)
@ -60,26 +129,41 @@ class _PROXY_MaxIterationsHandler(CustomLogger):
- session_id: sent by caller via x-litellm-session-id header or
metadata.session_id in request body
Cache key pattern:
{session_iterations:<session_id>}:count
Cache keys:
{session_iterations:<session_id>}:count (legacy aggregate)
{session_iterations:<session_id>}:agent-scope (per-agent migration ledger)
Unbound keys use only the legacy aggregate.
Multi-instance support:
Uses Redis Lua script for atomic increment (same pattern as
parallel_request_limiter_v3). Falls back to in-memory cache
when Redis is unavailable.
Uses Redis Lua scripts for atomic increments when Redis is configured.
Uses process-local memory only when Redis is not configured; Redis
errors propagate so a failed shared counter cannot silently bypass the
limit.
"""
def __init__(self, internal_usage_cache: InternalUsageCache):
self.internal_usage_cache = internal_usage_cache
self._local_lock = asyncio.Lock()
self.increment_script: Callable[..., Awaitable[object]] | None = None
self._registered_redis_cache: object | None = None
self.ttl = int(os.getenv("LITELLM_MAX_ITERATIONS_TTL", DEFAULT_MAX_ITERATIONS_TTL))
# Register Lua script with Redis if available (same pattern as v3 limiter)
if self.internal_usage_cache.dual_cache.redis_cache is not None:
self.increment_script = self.internal_usage_cache.dual_cache.redis_cache.async_register_script(
MAX_ITERATIONS_INCREMENT_SCRIPT
)
else:
self._ensure_redis_scripts()
def _ensure_redis_scripts(self) -> None:
redis_cache: Final = self.internal_usage_cache.dual_cache.redis_cache
if redis_cache is None:
self.increment_script = None
self._registered_redis_cache = None
return
if redis_cache is self._registered_redis_cache:
return
self.increment_script = cast( # cast-ok: Redis registration returns the callable invoked below
Callable[..., Awaitable[object]],
redis_cache.async_register_script(MAX_ITERATIONS_INCREMENT_SCRIPT),
)
self._registered_redis_cache = redis_cache
@with_service_target("session_iterations")
async def async_pre_call_hook(
@ -111,8 +195,11 @@ class _PROXY_MaxIterationsHandler(CustomLogger):
)
# Increment and check
cache_key: Final = self._make_cache_key(session_id)
current_count: Final = await self._increment_and_get(cache_key)
current_count: Final = (
await self._increment_legacy_and_get(self._make_legacy_cache_key(session_id))
if user_api_key_dict.agent_id is None
else await self._increment_agent_and_get(session_id, user_api_key_dict.agent_id)
)
if current_count > max_iterations:
resolved_model, llm_provider = resolve_llm_provider_for_rate_limit(data.get("model") if data else None)
@ -173,54 +260,137 @@ class _PROXY_MaxIterationsHandler(CustomLogger):
return int(max_iterations)
return None
def _make_cache_key(self, session_id: str) -> str:
"""
Create cache key for session iteration counter.
Uses Redis hash-tag pattern {session_iterations:<session_id>} so all
keys for a session land on the same Redis Cluster slot.
"""
def _make_legacy_cache_key(self, session_id: str) -> str:
return f"{{session_iterations:{session_id}}}:count"
async def _increment_and_get(self, cache_key: str) -> int:
"""
Atomically increment the session counter and return the new value.
def _make_agent_scope_cache_key(self, session_id: str) -> str:
return f"{{session_iterations:{session_id}}}:agent-scope"
Tries Redis first (via registered Lua script for atomicity across
instances), falls back to in-memory cache.
"""
def _make_agent_scope_field(self, agent_id: str) -> str:
from litellm.proxy.agent_endpoints.agent_registry import global_agent_registry
stable_agent_id: Final = json.dumps(global_agent_registry.stable_agent_id(agent_id), separators=(",", ":"))
return f"agent:{stable_agent_id}"
async def _get_local_scope(self, cache_key: str) -> Mapping[str, object] | None:
result: Final[object | None] = cast( # cast-ok: cache API returns Any; validate value before use
object | None,
await self.internal_usage_cache.async_get_cache(
key=cache_key,
litellm_parent_otel_span=None,
local_only=True,
),
)
if result is None:
return None
if isinstance(result, dict) and all(
isinstance(key, str)
for key in cast(Mapping[object, object], result) # cast-ok: dict shape is checked and keys remain opaque
):
return cast(Mapping[str, object], result) # cast-ok: keys are checked and values remain opaque
raise RuntimeError("Agent session scope cache has an invalid value")
@staticmethod
def _parse_count(value: object) -> int:
if isinstance(value, bool) or not isinstance(value, (int, float, str, bytes)):
raise TypeError("Agent session count is invalid")
if isinstance(value, float) and (not math.isfinite(value) or not value.is_integer()):
raise RuntimeError("Agent session count is not an integer")
count: Final = int(value)
if count < 0:
raise RuntimeError("Agent session count is negative")
return count
async def _get_local_count(self, cache_key: str) -> int | None:
local_result: Final[object | None] = cast( # cast-ok: cache API returns Any; validate value below
object | None,
await self.internal_usage_cache.async_get_cache(
key=cache_key,
litellm_parent_otel_span=None,
local_only=True,
),
)
return None if local_result is None else self._parse_count(local_result)
async def _increment_legacy_and_get(self, cache_key: str) -> int:
self._ensure_redis_scripts()
if self.increment_script is not None:
result: Final[object] = await self.increment_script(
keys=[cache_key],
args=[self.ttl],
)
if isinstance(result, (int, float, str, bytes)):
return int(result)
raise TypeError(f"Unexpected Redis iteration result: {type(result).__name__}")
async with self._local_lock:
current: Final = await self._get_local_count(cache_key)
new_value: Final = (current or 0) + 1
await self.internal_usage_cache.async_set_cache(
key=cache_key,
value=new_value,
ttl=self.ttl,
litellm_parent_otel_span=None,
local_only=True,
)
return new_value
async def _increment_agent_and_get(self, session_id: str, agent_id: str) -> int:
legacy_key: Final = self._make_legacy_cache_key(session_id)
agent_scope_key: Final = self._make_agent_scope_cache_key(session_id)
agent_field: Final = self._make_agent_scope_field(agent_id)
self._ensure_redis_scripts()
if self.increment_script is not None:
try:
result: Final = await self.increment_script(
keys=[cache_key],
args=[self.ttl],
result: Final[object] = await self.increment_script(
keys=[legacy_key, agent_scope_key],
args=[self.ttl, agent_field],
)
return int(result)
if isinstance(result, (int, float, str, bytes)):
return int(result)
raise TypeError(f"Unexpected Redis iteration result: {type(result).__name__}")
except Exception as e:
verbose_proxy_logger.warning(
"MaxIterationsHandler: Redis failed, falling back to in-memory: %s",
"MaxIterationsHandler: Redis migration increment failed; refusing an unsafe retry: %s",
str(e),
)
raise
# Fallback: in-memory cache
return await self._in_memory_increment(cache_key)
async def _in_memory_increment(self, cache_key: str) -> int:
"""Increment counter in in-memory cache with TTL."""
current: Final = await self.internal_usage_cache.async_get_cache(
key=cache_key,
litellm_parent_otel_span=None,
local_only=True,
)
new_value: Final = (int(current) if current is not None else 0) + 1
await self.internal_usage_cache.async_set_cache(
key=cache_key,
value=new_value,
ttl=self.ttl,
litellm_parent_otel_span=None,
local_only=True,
)
return new_value
async with self._local_lock:
cached_legacy_value: Final = await self._get_local_count(legacy_key)
agent_scope: Final = await self._get_local_scope(agent_scope_key)
if cached_legacy_value is None and agent_scope is not None:
raise RuntimeError("Legacy session count expired before agent scope")
legacy_value: Final = 0 if cached_legacy_value is None else cached_legacy_value
raw_total_new: Final = agent_scope.get("__total_new") if agent_scope is not None else 0
if raw_total_new is None:
raise RuntimeError("Agent session scope is missing its migration total")
raw_agent_value: Final = agent_scope.get(agent_field, 0) if agent_scope is not None else 0
total_new_value: Final = self._parse_count(raw_total_new)
agent_value: Final = self._parse_count(raw_agent_value)
new_legacy: Final = legacy_value + 1
new_total_new: Final = total_new_value + 1
new_agent: Final = agent_value + 1
await self.internal_usage_cache.async_set_cache(
key=legacy_key,
value=new_legacy,
ttl=self.ttl,
litellm_parent_otel_span=None,
local_only=True,
)
await self.internal_usage_cache.async_set_cache(
key=agent_scope_key,
value={
**(agent_scope or {}),
"__total_new": new_total_new,
agent_field: new_agent,
},
# Keep this map on a shorter rolling TTL than the aggregate fallback.
ttl=max(self.ttl - 1, 0),
litellm_parent_otel_span=None,
local_only=True,
)
return max(new_legacy - new_total_new, 0) + new_agent
PROXY_MaxIterationsHandler: Final = _PROXY_MaxIterationsHandler

View file

@ -0,0 +1,309 @@
import asyncio
import json
import os
import uuid
from typing import Final
from unittest.mock import patch
import pytest
from redis import Redis
from redis.exceptions import ResponseError
from litellm.caching.caching import DualCache
from litellm.caching.redis_cache import RedisCache
from litellm.proxy.agent_endpoints.agent_registry import AgentRegistry
from litellm.proxy.hooks.max_budget_per_session_limiter import PROXY_MaxBudgetPerSessionHandler
from litellm.proxy.hooks.max_iterations_limiter import PROXY_MaxIterationsHandler
from litellm.proxy.utils import InternalUsageCache
from litellm.types.agents import AgentResponse
from tests.integration._support.client import eventually
def _migration_agent(agent_id: str) -> AgentResponse:
return AgentResponse(
agent_id=agent_id,
agent_name=agent_id,
litellm_params={"max_iterations": 200, "max_budget_per_session": 10.0},
agent_card_params={"name": agent_id, "version": "1"},
)
@pytest.mark.asyncio
@pytest.mark.parametrize("agent_value", (" 0.5", "0.5 ", b"0.5\x00", b"0.5\x00junk"))
@pytest.mark.parametrize("operation", ("read", "increment"))
async def test_redis_budget_rejects_invalid_numeric_text_before_mutating_the_ledger(
agent_value: str | bytes, operation: str
) -> None:
host: Final = os.environ["REDIS_HOST"]
port: Final = int(os.environ["REDIS_PORT"])
redis_cache: Final = RedisCache(host=host, port=port)
handler: Final = PROXY_MaxBudgetPerSessionHandler(InternalUsageCache(DualCache(redis_cache=redis_cache)))
session_id: Final = f"whitespace-budget-{uuid.uuid4().hex}"
legacy_key: Final = redis_cache.check_and_fix_namespace(handler._make_legacy_cache_key(session_id))
scope_key: Final = redis_cache.check_and_fix_namespace(handler._make_agent_scope_cache_key(session_id))
agent_field: Final = handler._make_agent_scope_field("agent-a")
with Redis(host=host, port=port) as raw:
try:
raw.set(legacy_key, "0.5")
raw.hset(scope_key, mapping={"__total_new": "0.5", agent_field: agent_value})
previous_legacy: Final = raw.get(legacy_key)
previous_scope: Final = raw.hgetall(scope_key)
if operation == "read":
with pytest.raises(ResponseError):
await handler._get_agent_spend(session_id, "agent-a")
else:
with pytest.raises(ResponseError):
await handler._increment_agent_spend(session_id, "agent-a", 0.25)
assert raw.get(legacy_key) == previous_legacy
assert raw.hgetall(scope_key) == previous_scope
finally:
raw.delete(legacy_key, scope_key)
@pytest.mark.asyncio
@pytest.mark.parametrize("counter", ("legacy", "total_new", "agent"))
async def test_redis_iteration_rejects_leading_zero_before_mutating_the_ledger(counter: str) -> None:
host: Final = os.environ["REDIS_HOST"]
port: Final = int(os.environ["REDIS_PORT"])
redis_cache: Final = RedisCache(host=host, port=port)
handler: Final = PROXY_MaxIterationsHandler(InternalUsageCache(DualCache(redis_cache=redis_cache)))
session_id: Final = f"leading-zero-iterations-{uuid.uuid4().hex}"
legacy_key: Final = redis_cache.check_and_fix_namespace(handler._make_legacy_cache_key(session_id))
scope_key: Final = redis_cache.check_and_fix_namespace(handler._make_agent_scope_cache_key(session_id))
agent_field: Final = handler._make_agent_scope_field("agent-a")
with Redis(host=host, port=port) as raw:
try:
raw.set(legacy_key, "01" if counter == "legacy" else "1")
raw.hset(
scope_key,
mapping={
"__total_new": "01" if counter == "total_new" else "1",
agent_field: "01" if counter == "agent" else "1",
},
)
previous_legacy: Final = raw.get(legacy_key)
previous_scope: Final = raw.hgetall(scope_key)
with pytest.raises(ResponseError):
await handler._increment_agent_and_get(session_id, "agent-a")
assert raw.get(legacy_key) == previous_legacy
assert raw.hgetall(scope_key) == previous_scope
finally:
raw.delete(legacy_key, scope_key)
@pytest.mark.asyncio
@pytest.mark.parametrize("counter", ("legacy", "legacy_with_scope", "total_new", "agent"))
@pytest.mark.parametrize("invalid_value", ("bad", -2.0, float("nan"), float("inf")))
async def test_redis_budget_rejects_malformed_ledger(counter: str, invalid_value: object) -> None:
host: Final = os.environ["REDIS_HOST"]
port: Final = int(os.environ["REDIS_PORT"])
redis_cache: Final = RedisCache(host=host, port=port)
handler: Final = PROXY_MaxBudgetPerSessionHandler(InternalUsageCache(DualCache(redis_cache=redis_cache)))
session_id: Final = f"malformed-budget-{uuid.uuid4().hex}"
legacy_key: Final = redis_cache.check_and_fix_namespace(handler._make_legacy_cache_key(session_id))
scope_key: Final = redis_cache.check_and_fix_namespace(handler._make_agent_scope_cache_key(session_id))
agent_field: Final = handler._make_agent_scope_field("agent-a")
with Redis(host=host, port=port) as raw:
try:
raw.set(legacy_key, json.dumps(invalid_value) if counter.startswith("legacy") else "2.0")
if counter != "legacy":
raw.hset(
scope_key,
mapping={
"__total_new": json.dumps(invalid_value) if counter == "total_new" else "2.0",
agent_field: json.dumps(invalid_value) if counter == "agent" else "0.2",
},
)
with pytest.raises(ResponseError):
await handler._get_agent_spend(session_id, "agent-a")
with pytest.raises(ResponseError):
await handler._increment_agent_spend(session_id, "agent-a", 0.1)
finally:
raw.delete(legacy_key, scope_key)
@pytest.mark.asyncio
@pytest.mark.parametrize("invalid_value", ("bad", -5, 0.9, float("nan"), float("inf")))
async def test_redis_iteration_rejects_malformed_legacy_ledger(invalid_value: object) -> None:
host: Final = os.environ["REDIS_HOST"]
port: Final = int(os.environ["REDIS_PORT"])
redis_cache: Final = RedisCache(host=host, port=port)
handler: Final = PROXY_MaxIterationsHandler(InternalUsageCache(DualCache(redis_cache=redis_cache)))
session_id: Final = f"malformed-iterations-{uuid.uuid4().hex}"
legacy_key: Final = redis_cache.check_and_fix_namespace(handler._make_legacy_cache_key(session_id))
scope_key: Final = redis_cache.check_and_fix_namespace(handler._make_agent_scope_cache_key(session_id))
with Redis(host=host, port=port) as raw:
try:
raw.set(legacy_key, json.dumps(invalid_value))
with pytest.raises(ResponseError):
await handler._increment_agent_and_get(session_id, "agent-a")
finally:
raw.delete(legacy_key, scope_key)
@pytest.mark.asyncio
async def test_redis_budget_migration_is_atomic_and_preserves_session_ttl() -> None:
host: Final = os.environ["REDIS_HOST"]
port: Final = int(os.environ["REDIS_PORT"])
redis: Final = RedisCache(host=host, port=port)
redis_client: Final = redis.init_async_client()
handler: Final = PROXY_MaxBudgetPerSessionHandler(InternalUsageCache(DualCache(redis_cache=redis)))
handler.ttl = 30
registry: Final = AgentRegistry()
registry.register_agent(_migration_agent("agent-a"))
registry.register_agent(_migration_agent("agent-b"))
session_id: Final = f"budget-migration-{uuid.uuid4().hex}"
concurrent_session: Final = f"budget-concurrent-{uuid.uuid4().hex}"
with patch("litellm.proxy.agent_endpoints.agent_registry.global_agent_registry", registry):
try:
legacy_key: Final = handler._make_legacy_cache_key(session_id)
agent_scope_key: Final = handler._make_agent_scope_cache_key(session_id)
concurrent_legacy_key: Final = handler._make_legacy_cache_key(concurrent_session)
concurrent_scope_key: Final = handler._make_agent_scope_cache_key(concurrent_session)
keys: Final = (legacy_key, agent_scope_key, concurrent_legacy_key, concurrent_scope_key)
await redis.async_set_cache(key=legacy_key, value=0.5, ttl=30)
legacy_redis_key: Final = redis.check_and_fix_namespace(legacy_key)
initial_ttl: Final = await redis_client.pttl(legacy_redis_key)
await handler._increment_agent_spend(session_id, "agent-a", 0.4)
await redis.async_increment(legacy_key, 0.2, ttl=30) # Old pod updates only the aggregate.
await handler._increment_agent_spend(session_id, "agent-a", 0.1)
await handler._increment_agent_spend(session_id, "agent-b", 0.1)
assert await handler._get_agent_spend(session_id, "agent-a") == pytest.approx(1.2)
assert await handler._get_agent_spend(session_id, "agent-b") == pytest.approx(0.8)
hash_tags: Final = {key[key.index("{") : key.index("}") + 1] for key in keys[:2]}
assert len(hash_tags) == 1
ttls: Final = await redis_client.eval(
"return {redis.call('PTTL', KEYS[1]), redis.call('PTTL', KEYS[2])}",
2,
redis.check_and_fix_namespace(agent_scope_key),
legacy_redis_key,
)
assert 0 <= ttls[0] <= ttls[1]
assert ttls[1] <= initial_ttl
await redis_client.pexpire(legacy_redis_key, 15000)
await redis_client.pexpire(redis.check_and_fix_namespace(agent_scope_key), 14999)
before_increment_ttl: Final = await redis_client.pttl(legacy_redis_key)
before_increment_sidecar_ttl: Final = await redis_client.pttl(
redis.check_and_fix_namespace(agent_scope_key)
)
await handler._increment_agent_spend(session_id, "agent-a", 0.01)
after_increment_ttl: Final = await redis_client.pttl(legacy_redis_key)
after_increment_sidecar_ttl: Final = await redis_client.pttl(redis.check_and_fix_namespace(agent_scope_key))
assert after_increment_ttl <= before_increment_ttl
assert after_increment_sidecar_ttl <= before_increment_sidecar_ttl
# Losing the hash evicts migration and agent counters together; the aggregate remains a safe baseline.
await redis_client.delete(redis.check_and_fix_namespace(agent_scope_key))
assert await handler._get_agent_spend(session_id, "agent-b") == pytest.approx(1.31)
await handler._increment_agent_spend(session_id, "agent-a", 0.01)
assert await handler._get_agent_spend(session_id, "agent-a") == pytest.approx(1.32)
await redis.async_set_cache(key=concurrent_legacy_key, value=5.0, ttl=30)
await asyncio.gather(
*(handler._increment_agent_spend(concurrent_session, "agent-a", 0.01) for _ in range(100)),
*(handler._increment_agent_spend(concurrent_session, "agent-b", 0.01) for _ in range(50)),
)
assert await handler._get_agent_spend(concurrent_session, "agent-a") == pytest.approx(6.0)
assert await handler._get_agent_spend(concurrent_session, "agent-b") == pytest.approx(5.5)
saved_total_new: Final = await redis_client.hget(
redis.check_and_fix_namespace(agent_scope_key), "__total_new"
)
await redis_client.hdel(redis.check_and_fix_namespace(agent_scope_key), "__total_new")
with pytest.raises(ResponseError, match="agent session scope is missing its migration total"):
await handler._get_agent_spend(session_id, "agent-a")
assert saved_total_new is not None
await redis_client.hset(redis.check_and_fix_namespace(agent_scope_key), "__total_new", saved_total_new)
await redis_client.pexpire(legacy_redis_key, 100)
with Redis(host=host, port=port) as raw:
eventually(lambda: raw.exists(legacy_redis_key), lambda value: value == 0)
with pytest.raises(ResponseError, match="legacy session spend expired before agent scope"):
await handler._get_agent_spend(session_id, "agent-a")
finally:
await redis_client.delete(*(redis.check_and_fix_namespace(key) for key in keys))
@pytest.mark.asyncio
async def test_redis_iteration_migration_is_atomic_and_preserves_session_ttl() -> None:
host: Final = os.environ["REDIS_HOST"]
port: Final = int(os.environ["REDIS_PORT"])
redis: Final = RedisCache(host=host, port=port)
redis_client: Final = redis.init_async_client()
handler: Final = PROXY_MaxIterationsHandler(InternalUsageCache(DualCache(redis_cache=redis)))
handler.ttl = 30
registry: Final = AgentRegistry()
registry.register_agent(_migration_agent("agent-a"))
registry.register_agent(_migration_agent("agent-b"))
session_id: Final = f"iterations-migration-{uuid.uuid4().hex}"
with patch("litellm.proxy.agent_endpoints.agent_registry.global_agent_registry", registry):
try:
legacy_key: Final = handler._make_legacy_cache_key(session_id)
agent_scope_key: Final = handler._make_agent_scope_cache_key(session_id)
keys: Final = (legacy_key, agent_scope_key)
await redis.async_set_cache(key=legacy_key, value=3, ttl=30)
legacy_redis_key: Final = redis.check_and_fix_namespace(legacy_key)
initial_ttl: Final = await redis_client.pttl(legacy_redis_key)
hash_tags: Final = {key[key.index("{") : key.index("}") + 1] for key in keys}
assert len(hash_tags) == 1
assert await handler._increment_agent_and_get(session_id, "agent-a") == 4
assert await handler._increment_agent_and_get(session_id, "agent-b") == 4
await redis_client.incr(legacy_redis_key) # Old pod updates only the aggregate.
assert await handler._increment_agent_and_get(session_id, "agent-a") == 6
assert await handler._increment_agent_and_get(session_id, "agent-b") == 6
ttls: Final = await redis_client.eval(
"return {redis.call('PTTL', KEYS[1]), redis.call('PTTL', KEYS[2])}",
2,
redis.check_and_fix_namespace(agent_scope_key),
legacy_redis_key,
)
assert 0 <= ttls[0] <= ttls[1]
assert ttls[1] <= initial_ttl
await redis_client.pexpire(legacy_redis_key, 15000)
await redis_client.pexpire(redis.check_and_fix_namespace(agent_scope_key), 14999)
before_increment_ttl: Final = await redis_client.pttl(legacy_redis_key)
before_increment_sidecar_ttl: Final = await redis_client.pttl(
redis.check_and_fix_namespace(agent_scope_key)
)
assert await handler._increment_agent_and_get(session_id, "agent-a") == 7
after_increment_ttl: Final = await redis_client.pttl(legacy_redis_key)
after_increment_sidecar_ttl: Final = await redis_client.pttl(redis.check_and_fix_namespace(agent_scope_key))
assert after_increment_ttl <= before_increment_ttl
assert after_increment_sidecar_ttl <= before_increment_sidecar_ttl
values: Final = await asyncio.gather(
*(handler._increment_agent_and_get(session_id, "agent-a") for _ in range(100))
)
assert sorted(values) == list(range(8, 108))
# Losing the hash evicts migration and agent counters together; the aggregate remains a safe baseline.
await redis_client.delete(redis.check_and_fix_namespace(agent_scope_key))
assert await handler._increment_agent_and_get(session_id, "agent-b") == 110
assert await handler._increment_agent_and_get(session_id, "agent-a") == 110
saved_total_new: Final = await redis_client.hget(
redis.check_and_fix_namespace(agent_scope_key), "__total_new"
)
await redis_client.hdel(redis.check_and_fix_namespace(agent_scope_key), "__total_new")
with pytest.raises(ResponseError, match="agent session scope is missing its migration total"):
await handler._increment_agent_and_get(session_id, "agent-a")
assert saved_total_new is not None
await redis_client.hset(redis.check_and_fix_namespace(agent_scope_key), "__total_new", saved_total_new)
await redis_client.pexpire(legacy_redis_key, 100)
with Redis(host=host, port=port) as raw:
eventually(lambda: raw.exists(legacy_redis_key), lambda value: value == 0)
with pytest.raises(ResponseError, match="legacy session count expired before agent scope"):
await handler._increment_agent_and_get(session_id, "agent-a")
finally:
await redis_client.delete(*(redis.check_and_fix_namespace(key) for key in keys))

View file

@ -8,25 +8,30 @@ Tests that session-scoped budget tracking works correctly:
- Requests without agent_id pass through
"""
from unittest.mock import patch
import asyncio
import json
import logging
from typing import Final
from unittest.mock import AsyncMock, patch
import pytest
from fastapi import HTTPException
from litellm.caching.caching import DualCache
from litellm.caching.redis_cache import _redis_circuit_breaker_guard
from litellm.caching.redis_cache import RedisCircuitBreakerOpenError, _redis_circuit_breaker_guard
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.agent_endpoints.agent_registry import AgentRegistry
from litellm.proxy.hooks.max_budget_per_session_limiter import (
PROXY_MaxBudgetPerSessionHandler,
)
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup
from litellm.proxy.utils import InternalUsageCache
from litellm.types.agents import AgentResponse
def _make_mock_agent(max_budget_per_session: float) -> AgentResponse:
def _make_mock_agent(max_budget_per_session: float, agent_id: str = "agent-budget-123") -> AgentResponse:
return AgentResponse(
agent_id="agent-budget-123",
agent_id=agent_id,
agent_name="budget-agent",
litellm_params={"max_budget_per_session": max_budget_per_session},
agent_card_params={"name": "budget-agent", "version": "1.0.0"},
@ -49,11 +54,9 @@ async def test_budget_per_session_under_budget_passes():
mock_agent = _make_mock_agent(max_budget_per_session=5.0)
with patch(
"litellm.proxy.agent_endpoints.agent_registry.global_agent_registry"
) as mock_registry:
mock_registry.get_agent_by_id.return_value = mock_agent
registry: Final = AgentRegistry()
registry.register_agent(mock_agent)
with patch("litellm.proxy.agent_endpoints.agent_registry.global_agent_registry", registry):
result = await handler.async_pre_call_hook(
user_api_key_dict=user_api_key_dict,
cache=local_cache,
@ -79,16 +82,13 @@ async def test_budget_per_session_exceeds_budget():
)
session_id = "session-over-budget"
cache_key = handler._make_cache_key(session_id)
await handler._increment_spend(cache_key, 1.50)
await handler._increment_agent_spend(session_id, "agent-budget-123", 1.50)
mock_agent = _make_mock_agent(max_budget_per_session=1.0)
with patch(
"litellm.proxy.agent_endpoints.agent_registry.global_agent_registry"
) as mock_registry:
mock_registry.get_agent_by_id.return_value = mock_agent
registry: Final = AgentRegistry()
registry.register_agent(mock_agent)
with patch("litellm.proxy.agent_endpoints.agent_registry.global_agent_registry", registry):
with pytest.raises(HTTPException) as exc_info:
await handler.async_pre_call_hook(
user_api_key_dict=user_api_key_dict,
@ -115,16 +115,13 @@ async def test_budget_per_session_independent_sessions():
agent_id="agent-budget-123",
)
cache_key_a = handler._make_cache_key("session-A")
await handler._increment_spend(cache_key_a, 3.0)
await handler._increment_agent_spend("session-A", "agent-budget-123", 3.0)
mock_agent = _make_mock_agent(max_budget_per_session=2.0)
with patch(
"litellm.proxy.agent_endpoints.agent_registry.global_agent_registry"
) as mock_registry:
mock_registry.get_agent_by_id.return_value = mock_agent
registry: Final = AgentRegistry()
registry.register_agent(mock_agent)
with patch("litellm.proxy.agent_endpoints.agent_registry.global_agent_registry", registry):
# Session A should be blocked
with pytest.raises(HTTPException) as exc_info:
await handler.async_pre_call_hook(
@ -167,6 +164,438 @@ async def test_no_agent_id_passes():
assert result is None
@pytest.mark.asyncio
@pytest.mark.parametrize("response_cost", (0.125, 0.25))
async def test_concurrent_calls_are_admitted_before_their_costs_reach_the_stop_threshold(response_cost: float) -> None:
cache: Final = DualCache()
handler: Final = PROXY_MaxBudgetPerSessionHandler(InternalUsageCache(cache))
agent: Final = _make_mock_agent(1.0)
registry: Final = AgentRegistry()
registry.register_agent(agent)
key: Final = UserAPIKeyAuth(agent_id=agent.agent_id)
session_id: Final = "concurrent-admission"
data: Final = {"metadata": {"session_id": session_id}}
metadata: Final = {"session_id": session_id, "agent_id": agent.agent_id}
with patch("litellm.proxy.agent_endpoints.agent_registry.global_agent_registry", registry):
await handler.async_log_success_event(
{"litellm_params": {"metadata": metadata}, "response_cost": 0.75}, None, None, None
)
admitted: Final = await asyncio.gather(
handler.async_pre_call_hook(key, cache, data, ""),
handler.async_pre_call_hook(key, cache, data, ""),
)
assert admitted == [None, None]
assert await handler._get_agent_spend(session_id, agent.agent_id) == pytest.approx(0.75)
await asyncio.gather(
handler.async_log_success_event(
{"litellm_params": {"metadata": metadata}, "response_cost": response_cost}, None, None, None
),
handler.async_log_success_event(
{"litellm_params": {"metadata": metadata}, "response_cost": response_cost}, None, None, None
),
)
assert await handler._get_agent_spend(session_id, agent.agent_id) == pytest.approx(0.75 + 2 * response_cost)
with pytest.raises(HTTPException) as rejected:
await handler.async_pre_call_hook(key, cache, data, "")
assert rejected.value.status_code == 429
@pytest.mark.asyncio
@pytest.mark.parametrize(
"counter,invalid_value",
tuple(
(counter, invalid_value)
for counter in ("legacy", "total_new", "agent")
for invalid_value in ({}, "bad", " 0.5", "0.5 ", True, -1.0, float("nan"), float("inf"))
)
+ (("total_new", None), ("agent", None)),
)
async def test_malformed_budget_ledger_is_rejected(counter: str, invalid_value: object) -> None:
cache: Final = DualCache()
handler: Final = PROXY_MaxBudgetPerSessionHandler(InternalUsageCache(cache))
session_id: Final = "malformed-budget-ledger"
agent_field: Final = handler._make_agent_scope_field("agent-a")
legacy_value: Final = invalid_value if counter == "legacy" else 2.0
await cache.async_set_cache(
key=handler._make_legacy_cache_key(session_id),
value=json.dumps(legacy_value) if isinstance(legacy_value, str) else legacy_value,
)
if counter != "legacy":
await cache.async_set_cache(
key=handler._make_agent_scope_cache_key(session_id),
value={
"__total_new": invalid_value if counter == "total_new" else 0.0,
agent_field: invalid_value if counter == "agent" else 0.0,
},
)
with pytest.raises((RuntimeError, TypeError, ValueError)):
await handler._get_agent_spend(session_id, "agent-a")
with pytest.raises((RuntimeError, TypeError, ValueError)):
await handler._increment_agent_spend(session_id, "agent-a", 0.25)
@pytest.mark.asyncio
@pytest.mark.parametrize("counter", ("legacy", "total_new", "agent"))
async def test_overflowing_spend_increment_leaves_the_session_ledger_unchanged(counter: str) -> None:
cache: Final = DualCache()
handler: Final = PROXY_MaxBudgetPerSessionHandler(InternalUsageCache(cache))
session_id: Final = "overflowing-spend-increment"
legacy_key: Final = handler._make_legacy_cache_key(session_id)
scope_key: Final = handler._make_agent_scope_cache_key(session_id)
agent_field: Final = handler._make_agent_scope_field("agent-a")
legacy_value: Final = 1e308 if counter == "legacy" else 0.0
scope_value: Final = {
"__total_new": 1e308 if counter == "total_new" else 0.0,
agent_field: 1e308 if counter == "agent" else 0.0,
}
await cache.async_set_cache(key=legacy_key, value=legacy_value)
await cache.async_set_cache(key=scope_key, value=scope_value)
with pytest.raises(RuntimeError):
await handler._increment_agent_spend(session_id, "agent-a", 1e308)
assert await cache.async_get_cache(key=legacy_key) == legacy_value
assert await cache.async_get_cache(key=scope_key) == scope_value
@pytest.mark.asyncio
@pytest.mark.parametrize(
"amount,expected_error",
(
(True, TypeError),
(0.0, RuntimeError),
(-1.0, RuntimeError),
(float("nan"), RuntimeError),
(float("inf"), RuntimeError),
),
)
async def test_invalid_spend_increment_leaves_the_session_ledger_unchanged(
amount: float, expected_error: type[Exception]
) -> None:
cache: Final = DualCache()
handler: Final = PROXY_MaxBudgetPerSessionHandler(InternalUsageCache(cache))
session_id: Final = "invalid-spend-increment"
assert await handler._increment_agent_spend(session_id, "agent-a", 0.5) == pytest.approx(0.5)
with pytest.raises(expected_error):
await handler._increment_agent_spend(session_id, "agent-a", amount)
assert await handler._get_agent_spend(session_id, "agent-a") == pytest.approx(0.5)
assert await handler._get_agent_spend(session_id, "agent-b") == 0.0
@pytest.mark.asyncio
async def test_invoking_another_agent_charges_the_authenticated_agents_session_budget() -> None:
cache: Final = DualCache()
handler: Final = PROXY_MaxBudgetPerSessionHandler(InternalUsageCache(cache))
registry: Final = AgentRegistry()
registry.register_agent(_make_mock_agent(1.0, "actor-agent"))
registry.register_agent(_make_mock_agent(1.0, "target-agent"))
key: Final = UserAPIKeyAuth(agent_id="actor-agent").model_copy(update={"invoked_agent_id": "target-agent"})
data: Final = LiteLLMProxyRequestSetup.add_user_api_key_auth_to_request_metadata(
{"metadata": {"session_id": "agent-delegation"}}, key, "metadata"
)
assert data["metadata"]["agent_id"] == "target-agent"
assert data["metadata"]["billing_agent_id"] == "actor-agent"
with patch("litellm.proxy.agent_endpoints.agent_registry.global_agent_registry", registry):
assert await handler.async_pre_call_hook(key, cache, data, "") is None
await handler.async_log_success_event(
{"litellm_params": {"metadata": data["metadata"]}, "response_cost": 1.0}, None, None, None
)
with pytest.raises(HTTPException) as rejected:
await handler.async_pre_call_hook(key, cache, data, "")
assert rejected.value.status_code == 429
assert await handler._get_agent_spend("agent-delegation", "actor-agent") == pytest.approx(1.0)
assert await handler._get_agent_spend("agent-delegation", "target-agent") == 0.0
@pytest.mark.asyncio
@pytest.mark.parametrize(
"researcher_id,orchestrator_id,researcher_session,orchestrator_session",
[
("researcher", "orchestrator", "shared-trace", "shared-trace"),
("parent:child", "parent", "trace", "child:trace"),
],
)
async def test_agent_session_budget_counters_do_not_mix_usage(
researcher_id: str, orchestrator_id: str, researcher_session: str, orchestrator_session: str
) -> None:
cache: Final = DualCache()
handler: Final = PROXY_MaxBudgetPerSessionHandler(InternalUsageCache(cache))
registry: Final = AgentRegistry()
registry.register_agent(_make_mock_agent(3.0, researcher_id))
registry.register_agent(_make_mock_agent(1.0, orchestrator_id))
researcher: Final = UserAPIKeyAuth(agent_id=researcher_id)
orchestrator: Final = UserAPIKeyAuth(agent_id=orchestrator_id)
researcher_data: Final = {"metadata": {"session_id": researcher_session}}
orchestrator_data: Final = {"metadata": {"session_id": orchestrator_session}}
with patch("litellm.proxy.agent_endpoints.agent_registry.global_agent_registry", registry):
await handler.async_log_success_event(
{
"litellm_params": {"metadata": {"session_id": researcher_session, "agent_id": researcher_id}},
"response_cost": 2.0,
},
None,
None,
None,
)
assert await handler.async_pre_call_hook(researcher, cache, researcher_data, "") is None
assert await handler.async_pre_call_hook(orchestrator, cache, orchestrator_data, "") is None
await handler.async_log_success_event(
{
"litellm_params": {"metadata": {"session_id": orchestrator_session, "agent_id": orchestrator_id}},
"response_cost": 1.0,
},
None,
None,
None,
)
with pytest.raises(HTTPException) as rejected:
await handler.async_pre_call_hook(orchestrator, cache, orchestrator_data, "")
assert rejected.value.status_code == 429
assert "Current spend: $1.0000" in str(rejected.value.detail)
assert await handler.async_pre_call_hook(researcher, cache, researcher_data, "") is None
@pytest.mark.asyncio
async def test_legacy_agent_id_shares_the_registered_agents_budget() -> None:
cache: Final = DualCache()
handler: Final = PROXY_MaxBudgetPerSessionHandler(InternalUsageCache(cache))
registry: Final = AgentRegistry()
registry.load_agents_from_config(
(
{
"agent_name": "configured-agent",
"agent_card_params": {"name": "configured-agent", "version": "1"},
"litellm_params": {"max_budget_per_session": 1.0},
},
)
)
legacy_id, agent_id = next(iter(registry.config_agent_legacy_ids.items()))
with patch("litellm.proxy.agent_endpoints.agent_registry.global_agent_registry", registry):
await handler.async_log_success_event(
{"litellm_params": {"metadata": {"session_id": "trace", "agent_id": legacy_id}}, "response_cost": 1.0},
None,
None,
None,
)
for identity in (agent_id, legacy_id):
with pytest.raises(HTTPException) as rejected:
await handler.async_pre_call_hook(
UserAPIKeyAuth(agent_id=identity), cache, {"metadata": {"session_id": "trace"}}, ""
)
assert rejected.value.status_code == 429
assert "Current spend: $1.0000" in str(rejected.value.detail)
@pytest.mark.asyncio
async def test_agent_budget_keeps_spend_from_an_existing_session() -> None:
cache: Final = DualCache()
handler: Final = PROXY_MaxBudgetPerSessionHandler(InternalUsageCache(cache))
session_id: Final = "existing-session"
legacy_key: Final = handler._make_legacy_cache_key(session_id)
await cache.async_set_cache(key=legacy_key, value=1.0)
registry: Final = AgentRegistry()
registry.register_agent(_make_mock_agent(1.0))
with patch("litellm.proxy.agent_endpoints.agent_registry.global_agent_registry", registry):
with pytest.raises(HTTPException) as rejected:
await handler.async_pre_call_hook(
UserAPIKeyAuth(agent_id="agent-budget-123"),
cache,
{"metadata": {"session_id": session_id}},
"",
)
assert rejected.value.status_code == 429
assert "Current spend: $1.0000" in str(rejected.value.detail)
@pytest.mark.asyncio
async def test_agent_budget_carries_existing_spend_into_its_counter() -> None:
cache: Final = DualCache()
handler: Final = PROXY_MaxBudgetPerSessionHandler(InternalUsageCache(cache))
session_id: Final = "existing-session"
await cache.async_set_cache(key=handler._make_legacy_cache_key(session_id), value=0.75)
registry: Final = AgentRegistry()
registry.register_agent(_make_mock_agent(1.0))
kwargs: Final = {
"litellm_params": {"metadata": {"session_id": session_id, "agent_id": "agent-budget-123"}},
"response_cost": 0.1,
}
with patch("litellm.proxy.agent_endpoints.agent_registry.global_agent_registry", registry):
await handler.async_log_success_event(kwargs, None, None, None)
spend: Final = await handler._get_agent_spend(session_id, "agent-budget-123")
assert spend == pytest.approx(0.85)
@pytest.mark.asyncio
async def test_agent_budget_recovers_conservatively_after_scope_eviction() -> None:
cache: Final = DualCache()
handler: Final = PROXY_MaxBudgetPerSessionHandler(InternalUsageCache(cache))
session_id: Final = "scope-eviction-session"
registry: Final = AgentRegistry()
registry.register_agent(_make_mock_agent(10.0, "agent-a"))
registry.register_agent(_make_mock_agent(10.0, "agent-b"))
with patch("litellm.proxy.agent_endpoints.agent_registry.global_agent_registry", registry):
await handler._increment_agent_spend(session_id, "agent-a", 0.4)
await cache.async_delete_cache(key=handler._make_agent_scope_cache_key(session_id))
# Lost per-agent detail falls back to the aggregate, rather than resetting spend.
assert await handler._get_agent_spend(session_id, "agent-b") == pytest.approx(0.4)
assert await handler._increment_agent_spend(session_id, "agent-a", 0.1) == pytest.approx(0.5)
@pytest.mark.asyncio
async def test_agent_budget_fails_closed_when_scope_loses_migration_total() -> None:
cache: Final = DualCache()
handler: Final = PROXY_MaxBudgetPerSessionHandler(InternalUsageCache(cache))
session_id: Final = "inconsistent-scope-session"
registry: Final = AgentRegistry()
registry.register_agent(_make_mock_agent(10.0, "agent-a"))
scope_key: Final = handler._make_agent_scope_cache_key(session_id)
await cache.async_set_cache(key=handler._make_legacy_cache_key(session_id), value=0.4)
await cache.async_set_cache(key=scope_key, value={})
with patch("litellm.proxy.agent_endpoints.agent_registry.global_agent_registry", registry):
with pytest.raises(RuntimeError, match="missing its migration total"):
await handler._increment_agent_spend(session_id, "agent-a", 0.1)
@pytest.mark.asyncio
async def test_agent_budget_registers_redis_scripts_when_redis_is_attached_late() -> None:
class FakeRedisCache:
def __init__(self) -> None:
self.scripts: list[str] = []
self.calls: list[dict[str, object]] = []
def async_register_script(self, source: str):
self.scripts.append(source)
async def call(**kwargs: object) -> object:
self.calls.append(kwargs)
return "0.1"
return call
cache: Final = DualCache()
handler: Final = PROXY_MaxBudgetPerSessionHandler(InternalUsageCache(cache))
redis_cache: Final = FakeRedisCache()
cache.redis_cache = redis_cache
registry: Final = AgentRegistry()
registry.register_agent(_make_mock_agent(10.0, "agent-a"))
with patch("litellm.proxy.agent_endpoints.agent_registry.global_agent_registry", registry):
await handler._increment_agent_spend("late-redis", "agent-a", 0.1)
assert await handler._get_agent_spend("late-redis", "agent-a") == pytest.approx(0.1)
assert len(redis_cache.scripts) == 2
assert len(redis_cache.calls) == 2
assert all(len(call["keys"]) == 2 for call in redis_cache.calls)
@pytest.mark.asyncio
async def test_agent_budget_tracks_old_and_new_pods_without_mixing_new_agent_spend() -> None:
cache: Final = DualCache()
handler: Final = PROXY_MaxBudgetPerSessionHandler(InternalUsageCache(cache))
session_id: Final = "mixed-rollout-session"
legacy_key: Final = handler._make_legacy_cache_key(session_id)
await cache.async_set_cache(key=legacy_key, value=0.5)
registry: Final = AgentRegistry()
registry.register_agent(_make_mock_agent(3.0, "agent-a"))
registry.register_agent(_make_mock_agent(3.0, "agent-b"))
with patch("litellm.proxy.agent_endpoints.agent_registry.global_agent_registry", registry):
await handler._increment_agent_spend(session_id, "agent-a", 0.4)
assert await handler._get_agent_spend(session_id, "agent-a") == pytest.approx(0.9)
assert await handler._get_agent_spend(session_id, "agent-b") == pytest.approx(0.5)
# An old pod still writes only the legacy aggregate.
await cache.async_set_cache(key=legacy_key, value=1.1)
await handler._increment_agent_spend(session_id, "agent-a", 0.1)
await handler._increment_agent_spend(session_id, "agent-b", 0.1)
assert await handler._get_agent_spend(session_id, "agent-a") == pytest.approx(1.2)
assert await handler._get_agent_spend(session_id, "agent-b") == pytest.approx(0.8)
@pytest.mark.asyncio
async def test_agent_spend_is_recorded_before_a_budget_is_configured() -> None:
cache: Final = DualCache()
handler: Final = PROXY_MaxBudgetPerSessionHandler(InternalUsageCache(cache))
session_id: Final = "unlimited-session"
agent: Final = _make_mock_agent(1.0)
agent.litellm_params = {}
registry: Final = AgentRegistry()
registry.register_agent(agent)
with patch("litellm.proxy.agent_endpoints.agent_registry.global_agent_registry", registry):
await handler.async_log_success_event(
{
"litellm_params": {
"metadata": {"session_id": session_id, "agent_id": agent.agent_id},
},
"response_cost": 1.25,
},
None,
None,
None,
)
agent.litellm_params["max_budget_per_session"] = 1.0
with pytest.raises(HTTPException) as rejected:
await handler.async_pre_call_hook(
UserAPIKeyAuth(agent_id=agent.agent_id),
cache,
{"metadata": {"session_id": session_id}},
"",
)
assert rejected.value.status_code == 429
assert "Current spend: $1.2500" in str(rejected.value.detail)
@pytest.mark.asyncio
async def test_agent_budget_redis_errors_are_not_retried_or_read_locally() -> None:
cache: Final = DualCache()
handler: Final = PROXY_MaxBudgetPerSessionHandler(InternalUsageCache(cache))
redis_cache: Final = object()
cache.redis_cache = redis_cache
handler._registered_redis_cache = redis_cache
increment_calls = 0
read_calls = 0
async def fail_increment(**_kwargs: object) -> object:
nonlocal increment_calls
increment_calls += 1
raise TimeoutError("reply timed out after Redis may have applied the script")
async def fail_read(**_kwargs: object) -> object:
nonlocal read_calls
read_calls += 1
raise TimeoutError("Redis read timed out")
handler.increment_script = fail_increment
handler.get_agent_spend_script = fail_read
with patch.object(handler, "_get_local_spend", new=AsyncMock(side_effect=AssertionError("must fail closed"))):
with pytest.raises(TimeoutError):
await handler._increment_agent_spend("uncertain-session", "agent-a", 0.1)
with pytest.raises(TimeoutError):
await handler._get_agent_spend("uncertain-session", "agent-a")
assert increment_calls == 1
assert read_calls == 1
class _OpenBreakerRedis:
def __init__(self) -> None:
from litellm.caching.redis_cache import RedisCircuitBreaker
@ -188,14 +617,16 @@ class _OpenBreakerRedis:
@pytest.mark.asyncio
async def test_an_open_circuit_breaker_reads_session_spend_locally_without_a_warning(caplog):
async def test_an_open_circuit_breaker_rejects_shared_spend_reads_without_a_warning(
caplog: pytest.LogCaptureFixture,
) -> None:
cache = DualCache(redis_cache=_OpenBreakerRedis()) # pyright: ignore[reportArgumentType] # duck-typed Redis double
handler = PROXY_MaxBudgetPerSessionHandler(internal_usage_cache=InternalUsageCache(cache))
caplog.clear()
with caplog.at_level(logging.DEBUG, logger="LiteLLM Proxy"):
spend = await handler._get_current_spend("{session_budget:quiet}:spend")
with pytest.raises(RedisCircuitBreakerOpenError):
await handler._get_agent_spend("quiet", "agent-a")
assert spend == 0.0
assert [record.getMessage() for record in caplog.records if record.levelno >= logging.WARNING] == []
assert any("circuit breaker is open" in record.getMessage() for record in caplog.records)

View file

@ -6,21 +6,24 @@ Tests that session-scoped iteration counting works correctly:
- Different sessions have independent counters
"""
from unittest.mock import MagicMock, patch
import asyncio
from typing import Final
from unittest.mock import AsyncMock, patch
import pytest
from fastapi import HTTPException
from litellm.caching.caching import DualCache
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.agent_endpoints.agent_registry import AgentRegistry
from litellm.proxy.hooks.max_iterations_limiter import PROXY_MaxIterationsHandler
from litellm.proxy.utils import InternalUsageCache
from litellm.types.agents import AgentResponse
def _make_mock_agent(max_iterations: int) -> AgentResponse:
def _make_mock_agent(max_iterations: int, agent_id: str = "agent-test-123") -> AgentResponse:
return AgentResponse(
agent_id="agent-test-123",
agent_id=agent_id,
agent_name="test-agent",
litellm_params={"max_iterations": max_iterations},
agent_card_params={"name": "test-agent", "version": "1.0.0"},
@ -46,9 +49,9 @@ async def test_max_iterations_basic_enforcement():
mock_agent = _make_mock_agent(max_iterations=3)
with patch("litellm.proxy.agent_endpoints.agent_registry.global_agent_registry") as mock_registry:
mock_registry.get_agent_by_id.return_value = mock_agent
registry: Final = AgentRegistry()
registry.register_agent(mock_agent)
with patch("litellm.proxy.agent_endpoints.agent_registry.global_agent_registry", registry):
# First 3 requests should succeed
for i in range(3):
await handler.async_pre_call_hook(
@ -89,9 +92,9 @@ async def test_max_iterations_different_sessions_independent():
mock_agent = _make_mock_agent(max_iterations=2)
with patch("litellm.proxy.agent_endpoints.agent_registry.global_agent_registry") as mock_registry:
mock_registry.get_agent_by_id.return_value = mock_agent
registry: Final = AgentRegistry()
registry.register_agent(mock_agent)
with patch("litellm.proxy.agent_endpoints.agent_registry.global_agent_registry", registry):
# Session A: 2 calls succeed
for _ in range(2):
await handler.async_pre_call_hook(
@ -150,3 +153,257 @@ async def test_max_iterations_no_agent_id_passes():
call_type="",
)
assert result is None
@pytest.mark.asyncio
@pytest.mark.parametrize(
"researcher_id,orchestrator_id,researcher_session,orchestrator_session",
[
("researcher", "orchestrator", "shared-trace", "shared-trace"),
("parent:child", "parent", "trace", "child:trace"),
],
)
async def test_agent_session_iteration_counters_do_not_mix_usage(
researcher_id: str, orchestrator_id: str, researcher_session: str, orchestrator_session: str
) -> None:
cache: Final = DualCache()
handler: Final = PROXY_MaxIterationsHandler(InternalUsageCache(cache))
registry: Final = AgentRegistry()
registry.register_agent(_make_mock_agent(4, researcher_id))
registry.register_agent(_make_mock_agent(2, orchestrator_id))
researcher: Final = UserAPIKeyAuth(agent_id=researcher_id)
orchestrator: Final = UserAPIKeyAuth(agent_id=orchestrator_id)
researcher_data: Final = {"metadata": {"session_id": researcher_session}}
orchestrator_data: Final = {"metadata": {"session_id": orchestrator_session}}
with patch("litellm.proxy.agent_endpoints.agent_registry.global_agent_registry", registry):
for _ in range(3):
assert await handler.async_pre_call_hook(researcher, cache, researcher_data, "") is None
for _ in range(2):
assert await handler.async_pre_call_hook(orchestrator, cache, orchestrator_data, "") is None
with pytest.raises(HTTPException) as rejected:
await handler.async_pre_call_hook(orchestrator, cache, orchestrator_data, "")
assert rejected.value.status_code == 429
assert "Current count: 3" in str(rejected.value.detail)
assert await handler.async_pre_call_hook(researcher, cache, researcher_data, "") is None
with pytest.raises(HTTPException) as researcher_rejected:
await handler.async_pre_call_hook(researcher, cache, researcher_data, "")
assert researcher_rejected.value.status_code == 429
assert "Current count: 5" in str(researcher_rejected.value.detail)
@pytest.mark.asyncio
async def test_key_metadata_iteration_limit_keeps_existing_session_count() -> None:
cache: Final = DualCache()
handler: Final = PROXY_MaxIterationsHandler(InternalUsageCache(cache))
key: Final = UserAPIKeyAuth(metadata={"max_iterations": 2})
await cache.async_set_cache(key="{session_iterations:existing}:count", value=2)
with pytest.raises(HTTPException) as rejected:
await handler.async_pre_call_hook(key, cache, {"metadata": {"session_id": "existing"}}, "")
assert rejected.value.status_code == 429
assert "Current count: 3" in str(rejected.value.detail)
@pytest.mark.asyncio
async def test_legacy_agent_id_shares_the_registered_agents_iteration_limit() -> None:
cache: Final = DualCache()
handler: Final = PROXY_MaxIterationsHandler(InternalUsageCache(cache))
registry: Final = AgentRegistry()
registry.load_agents_from_config(
(
{
"agent_name": "configured-agent",
"agent_card_params": {"name": "configured-agent", "version": "1"},
"litellm_params": {"max_iterations": 2},
},
)
)
legacy_id, agent_id = next(iter(registry.config_agent_legacy_ids.items()))
data: Final = {"metadata": {"session_id": "trace"}}
with patch("litellm.proxy.agent_endpoints.agent_registry.global_agent_registry", registry):
for identity in (legacy_id, agent_id):
assert await handler.async_pre_call_hook(UserAPIKeyAuth(agent_id=identity), cache, data, "") is None
with pytest.raises(HTTPException) as rejected:
await handler.async_pre_call_hook(UserAPIKeyAuth(agent_id=legacy_id), cache, data, "")
assert rejected.value.status_code == 429
assert "Current count: 3" in str(rejected.value.detail)
@pytest.mark.asyncio
async def test_agent_iteration_limit_keeps_count_from_an_existing_session() -> None:
cache: Final = DualCache()
handler: Final = PROXY_MaxIterationsHandler(InternalUsageCache(cache))
session_id: Final = "existing-session"
legacy_key: Final = handler._make_legacy_cache_key(session_id)
await cache.async_set_cache(key=legacy_key, value=2)
registry: Final = AgentRegistry()
registry.register_agent(_make_mock_agent(2))
with patch("litellm.proxy.agent_endpoints.agent_registry.global_agent_registry", registry):
with pytest.raises(HTTPException) as rejected:
await handler.async_pre_call_hook(
UserAPIKeyAuth(agent_id="agent-test-123"),
cache,
{"metadata": {"session_id": session_id}},
"",
)
assert rejected.value.status_code == 429
assert "Current count: 3" in str(rejected.value.detail)
legacy_count: Final = await cache.async_get_cache(key=handler._make_legacy_cache_key(session_id))
scope: Final = await cache.async_get_cache(key=handler._make_agent_scope_cache_key(session_id))
agent_field: Final = handler._make_agent_scope_field("agent-test-123")
assert isinstance(scope, dict)
assert legacy_count - scope["__total_new"] + scope[agent_field] == 3
@pytest.mark.asyncio
async def test_agent_iteration_recovers_conservatively_after_scope_eviction() -> None:
cache: Final = DualCache()
handler: Final = PROXY_MaxIterationsHandler(InternalUsageCache(cache))
session_id: Final = "scope-eviction-session"
registry: Final = AgentRegistry()
registry.register_agent(_make_mock_agent(20, "agent-a"))
registry.register_agent(_make_mock_agent(20, "agent-b"))
with patch("litellm.proxy.agent_endpoints.agent_registry.global_agent_registry", registry):
assert await handler._increment_agent_and_get(session_id, "agent-a") == 1
await cache.async_delete_cache(key=handler._make_agent_scope_cache_key(session_id))
# With the migration scope gone, prior usage becomes a conservative shared baseline.
assert await handler._increment_agent_and_get(session_id, "agent-b") == 2
assert await handler._increment_agent_and_get(session_id, "agent-a") == 2
@pytest.mark.asyncio
async def test_agent_iteration_fails_closed_when_scope_loses_migration_total() -> None:
cache: Final = DualCache()
handler: Final = PROXY_MaxIterationsHandler(InternalUsageCache(cache))
session_id: Final = "inconsistent-scope-session"
scope_key: Final = handler._make_agent_scope_cache_key(session_id)
await cache.async_set_cache(key=handler._make_legacy_cache_key(session_id), value=1)
await cache.async_set_cache(key=scope_key, value={'agent:"agent-a"': 1})
with pytest.raises(RuntimeError, match="missing its migration total"):
await handler._increment_agent_and_get(session_id, "agent-a")
@pytest.mark.asyncio
@pytest.mark.parametrize(
"counter,invalid_value",
tuple(
(counter, invalid_value)
for counter in ("legacy", "total_new", "agent")
for invalid_value in ({}, "bad", True, -1, 0.9, float("nan"), float("inf"))
)
+ (("total_new", None), ("agent", None)),
)
async def test_malformed_iteration_ledger_is_rejected(counter: str, invalid_value: object) -> None:
cache: Final = DualCache()
handler: Final = PROXY_MaxIterationsHandler(InternalUsageCache(cache))
session_id: Final = "malformed-iteration-ledger"
agent_field: Final = handler._make_agent_scope_field("agent-a")
await cache.async_set_cache(
key=handler._make_legacy_cache_key(session_id), value=invalid_value if counter == "legacy" else 2
)
if counter != "legacy":
await cache.async_set_cache(
key=handler._make_agent_scope_cache_key(session_id),
value={
"__total_new": invalid_value if counter == "total_new" else 0,
agent_field: invalid_value if counter == "agent" else 0,
},
)
with pytest.raises((RuntimeError, TypeError, ValueError)):
await handler._increment_agent_and_get(session_id, "agent-a")
@pytest.mark.asyncio
async def test_integral_json_float_iteration_counters_are_preserved() -> None:
cache: Final = DualCache()
handler: Final = PROXY_MaxIterationsHandler(InternalUsageCache(cache))
session_id: Final = "integral-json-float"
agent_field: Final = handler._make_agent_scope_field("agent-a")
await cache.async_set_cache(key=handler._make_legacy_cache_key(session_id), value=2.0)
await cache.async_set_cache(
key=handler._make_agent_scope_cache_key(session_id), value={"__total_new": 1.0, agent_field: 1.0}
)
assert await handler._increment_agent_and_get(session_id, "agent-a") == 3
@pytest.mark.asyncio
async def test_agent_iteration_registers_redis_scripts_when_redis_is_attached_late() -> None:
class FakeRedisCache:
def __init__(self) -> None:
self.scripts: list[str] = []
self.calls: list[dict[str, object]] = []
def async_register_script(self, source: str):
self.scripts.append(source)
async def call(**kwargs: object) -> object:
self.calls.append(kwargs)
return 1
return call
cache: Final = DualCache()
handler: Final = PROXY_MaxIterationsHandler(InternalUsageCache(cache))
redis_cache: Final = FakeRedisCache()
cache.redis_cache = redis_cache
registry: Final = AgentRegistry()
registry.register_agent(_make_mock_agent(10, "agent-a"))
with patch("litellm.proxy.agent_endpoints.agent_registry.global_agent_registry", registry):
assert await handler._increment_agent_and_get("late-redis", "agent-a") == 1
assert len(redis_cache.scripts) == 1
assert len(redis_cache.calls) == 1
assert len(redis_cache.calls[0]["keys"]) == 2
@pytest.mark.asyncio
async def test_agent_iteration_counter_tracks_old_and_new_pods_independently() -> None:
cache: Final = DualCache()
handler: Final = PROXY_MaxIterationsHandler(InternalUsageCache(cache))
session_id: Final = "mixed-rollout-session"
legacy_key: Final = handler._make_legacy_cache_key(session_id)
await cache.async_set_cache(key=legacy_key, value=3)
registry: Final = AgentRegistry()
registry.register_agent(_make_mock_agent(20, "agent-a"))
registry.register_agent(_make_mock_agent(20, "agent-b"))
with patch("litellm.proxy.agent_endpoints.agent_registry.global_agent_registry", registry):
assert await handler._increment_agent_and_get(session_id, "agent-a") == 4
assert await handler._increment_agent_and_get(session_id, "agent-b") == 4
# An old pod still writes only the legacy aggregate.
await cache.async_set_cache(key=legacy_key, value=6)
assert await handler._increment_agent_and_get(session_id, "agent-a") == 6
assert await handler._increment_agent_and_get(session_id, "agent-b") == 6
@pytest.mark.asyncio
async def test_agent_iteration_redis_error_is_not_retried_or_counted_locally() -> None:
cache: Final = DualCache()
handler: Final = PROXY_MaxIterationsHandler(InternalUsageCache(cache))
redis_cache: Final = object()
cache.redis_cache = redis_cache
handler._registered_redis_cache = redis_cache
calls = 0
async def fail_after_attempt(**_kwargs: object) -> object:
nonlocal calls
calls += 1
raise TimeoutError("reply timed out after Redis may have applied the script")
handler.increment_script = fail_after_attempt
with patch.object(handler, "_get_local_count", new=AsyncMock(side_effect=AssertionError("must fail closed"))):
with pytest.raises(TimeoutError):
await handler._increment_agent_and_get("uncertain-session", "agent-a")
assert calls == 1

View file

@ -43,6 +43,7 @@ import litellm
from litellm.caching.caching import DualCache
from litellm.exceptions import RateLimitError
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
from litellm.proxy.hooks.batch_rate_limiter import (
BatchFileUsage,
PROXY_BatchRateLimiter,
@ -61,7 +62,6 @@ from litellm.proxy.hooks.parallel_request_limiter import (
from litellm.proxy.hooks.parallel_request_limiter_v3 import (
PROXY_MaxParallelRequestsHandler_v3,
)
from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
from litellm.proxy.hooks.rate_limiter_utils import (
PROXY_LLM_PROVIDER_FALLBACK,
resolve_llm_provider_for_rate_limit,
@ -69,7 +69,6 @@ from litellm.proxy.hooks.rate_limiter_utils import (
from litellm.proxy.utils import InternalUsageCache
from litellm.types.agents import AgentResponse
# ---------------------------------------------------------------------------
# Helper class itself
# ---------------------------------------------------------------------------
@ -830,6 +829,7 @@ async def test_max_iterations_limiter_populates_provider():
with patch("litellm.proxy.agent_endpoints.agent_registry.global_agent_registry") as mock_registry:
mock_registry.get_agent_by_id.return_value = _make_iter_agent(max_iterations=1)
mock_registry.stable_agent_id.return_value = "agent-iter"
await handler.async_pre_call_hook(
user_api_key_dict=user_api_key_dict,
@ -867,6 +867,7 @@ async def test_max_iterations_limiter_unknown_model_falls_back():
with patch("litellm.proxy.agent_endpoints.agent_registry.global_agent_registry") as mock_registry:
mock_registry.get_agent_by_id.return_value = _make_iter_agent(max_iterations=1)
mock_registry.stable_agent_id.return_value = "agent-iter"
await handler.async_pre_call_hook(
user_api_key_dict=user_api_key_dict,
@ -913,7 +914,8 @@ async def test_max_budget_per_session_limiter_populates_provider():
with patch("litellm.proxy.agent_endpoints.agent_registry.global_agent_registry") as mock_registry:
mock_registry.get_agent_by_id.return_value = _make_session_budget_agent(max_budget=1.0)
with patch.object(handler, "_get_current_spend", new=AsyncMock(return_value=5.0)):
mock_registry.stable_agent_id.return_value = "agent-session-budget"
with patch.object(handler, "_get_agent_spend", new=AsyncMock(return_value=5.0)):
with pytest.raises(HTTPException) as exc_info:
await handler.async_pre_call_hook(
user_api_key_dict=user_api_key_dict,
@ -938,7 +940,8 @@ async def test_max_budget_per_session_limiter_unknown_model_falls_back():
with patch("litellm.proxy.agent_endpoints.agent_registry.global_agent_registry") as mock_registry:
mock_registry.get_agent_by_id.return_value = _make_session_budget_agent(max_budget=1.0)
with patch.object(handler, "_get_current_spend", new=AsyncMock(return_value=5.0)):
mock_registry.stable_agent_id.return_value = "agent-session-budget"
with patch.object(handler, "_get_agent_spend", new=AsyncMock(return_value=5.0)):
with pytest.raises(HTTPException) as exc_info:
await handler.async_pre_call_hook(
user_api_key_dict=user_api_key_dict,

View file

@ -498,6 +498,7 @@ class TestProxyHooksActuallyRaiseProxyRateLimitError:
"litellm.proxy.agent_endpoints.agent_registry.global_agent_registry"
) as mock_registry:
mock_registry.get_agent_by_id.return_value = agent
mock_registry.stable_agent_id.return_value = "agent-iter-1"
# First call within budget.
await handler.async_pre_call_hook(
user_api_key_dict=user_api_key_dict,
@ -840,7 +841,8 @@ class TestProxyHooksActuallyRaiseProxyRateLimitError:
)
internal_cache = MagicMock()
internal_cache.async_get_cache = AsyncMock(return_value=10.0)
internal_cache.dual_cache = DualCache()
internal_cache.async_get_cache = AsyncMock(side_effect=[10.0, None])
handler = PROXY_MaxBudgetPerSessionHandler(
internal_usage_cache=internal_cache,
)
@ -854,6 +856,7 @@ class TestProxyHooksActuallyRaiseProxyRateLimitError:
"litellm.proxy.agent_endpoints.agent_registry.global_agent_registry"
) as mock_registry:
mock_registry.get_agent_by_id.return_value = agent
mock_registry.stable_agent_id.return_value = "agent-session-1"
with pytest.raises(ProxyRateLimitError) as exc_info:
await handler.async_pre_call_hook(
user_api_key_dict=user_api_key_dict,