This commit is contained in:
Mateo Wang 2026-10-27 21:11:45 +00:00 • committed by GitHub
commit d89d7150a9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
15 changed files with 1585 additions and 69 deletions

View file

@ -434,6 +434,7 @@ _POSTGRES_OPERATION_BY_CALL_TYPE: Final[Mapping[str, PostgresOperation]] = Mappi
"reset_budget_windows": PostgresOperation("select", None),
"write_budget_windows": PostgresOperation("update", None),
"roll_window_spend_row": PostgresOperation("update", None),
"open_window_spend_row": PostgresOperation("upsert", None),
"seed_window_spend": PostgresOperation("select", None),
"select_window_spend_rows": PostgresOperation("select", None),
"commit_window_spend_updates": PostgresOperation("upsert", None),

View file

@ -64,6 +64,16 @@ _ROLL_WINDOW_SPEND_SQL: Final = (
"AND window_start < ($4::timestamptz AT TIME ZONE 'UTC')"
)
_OPEN_WINDOW_SPEND_SQL: Final = (
'INSERT INTO "LiteLLM_BudgetWindowSpend" '
"(entity_type, entity_id, window_duration, window_start, spend, created_at, updated_at) "
"VALUES ($1, $2, $3, ($4::timestamptz AT TIME ZONE 'UTC'), 0, "
"($4::timestamptz AT TIME ZONE 'UTC'), ($4::timestamptz AT TIME ZONE 'UTC')) "
"ON CONFLICT (entity_type, entity_id, window_duration) DO UPDATE SET "
"window_start = EXCLUDED.window_start, spend = 0, updated_at = EXCLUDED.updated_at "
"WHERE \"LiteLLM_BudgetWindowSpend\".window_start <= ($5::timestamptz AT TIME ZONE 'UTC')"
)
_SEED_FROM_SPEND_LOGS_KEY_SQL: Final = (
"SELECT COALESCE(SUM(spend), 0.0) AS total, "
"COALESCE(SUM(spend) FILTER (WHERE \"startTime\" < ($3::timestamptz AT TIME ZONE 'UTC')), 0.0) AS before_batch "
@ -389,3 +399,29 @@ async def roll_window_spend_row(
to_naive_utc(new_window_start),
to_naive_utc(datetime.now(timezone.utc)),
)
async def open_window_spend_row(
prisma_client: "PrismaClient",
entity_type: str,
entity_id: str,
window_duration: str,
window_start: datetime,
replaces_window_start: datetime | None,
) -> None:
"""Start a window at `window_start` unless another writer already started one.
The row is created when it does not exist, and moved onto the new window only
while it still holds `replaces_window_start`, the ended window the caller saw.
A writer that lost the race leaves the winner's window in place, so reading
the row back gives every writer the same window start.
"""
async with db_span("open_window_spend_row", "LiteLLM_BudgetWindowSpend"):
await prisma_client.db.execute_raw(
_OPEN_WINDOW_SPEND_SQL,
entity_type,
entity_id,
window_duration,
to_naive_utc(window_start),
None if replaces_window_start is None else to_naive_utc(replaces_window_start),
)

View file

@ -0,0 +1,173 @@
from collections.abc import Awaitable, Callable
from dataclasses import dataclass
from datetime import datetime, timezone
from typing import TYPE_CHECKING, Final, Protocol, TypeAlias
from litellm._logging import verbose_proxy_logger
from litellm.caching.in_memory_cache import InMemoryCache
from litellm.proxy.db.budget_window_spend_writer import open_window_spend_row
from litellm.proxy.db.db_transaction_queue.window_spend_update_queue import (
WindowSpendTransaction,
build_window_spend_transaction,
)
from litellm.repositories.table_repositories import BudgetWindowSpendRepository
from litellm.router_strategy.budget_limiter import budget_window_seconds
if TYPE_CHECKING:
from litellm.proxy.utils import PrismaClient
MODEL_BUDGET_WINDOW_ENTITY_TYPE: Final = "model_max_budget"
_READ_TTL_SECONDS: Final = 5
@dataclass(frozen=True, slots=True)
class SharedWindow:
started_at: float
spend: float
def is_open(self, budget_duration: str, now: float) -> bool:
return now < self.started_at + budget_window_seconds(self.started_at, budget_duration)
class LocalTTLCache(Protocol):
def get_cache(self, key: str) -> object: ...
def set_cache(self, key: str, value: object, *, ttl: float) -> None: ...
class SharedWindows(Protocol):
async def read(self, spend_key: str, budget_duration: str) -> SharedWindow | None: ...
async def open(self, spend_key: str, budget_duration: str, now: float) -> float: ...
async def add(self, spend_key: str, budget_duration: str, started_at: float, spend: float) -> None: ...
PrismaClientSource: TypeAlias = Callable[[], "PrismaClient | None"]
WindowSpendSink: TypeAlias = Callable[[WindowSpendTransaction], Awaitable[None]]
def _proxy_prisma_client() -> "PrismaClient | None":
try:
from litellm.proxy.proxy_server import prisma_client
except ImportError:
return None
return prisma_client
async def _queue_on_proxy_spend_writer(transaction: WindowSpendTransaction) -> None:
from litellm.proxy.proxy_server import proxy_logging_obj
await proxy_logging_obj.db_spend_update_writer.window_spend_update_queue.add_update(transaction)
@dataclass(frozen=True, slots=True)
class _Reading:
window: SharedWindow | None
class DatabaseSharedWindows:
def __init__(
self,
prisma_client_source: PrismaClientSource = _proxy_prisma_client,
sink: WindowSpendSink = _queue_on_proxy_spend_writer,
read_ttl_seconds: int = _READ_TTL_SECONDS,
) -> None:
self._prisma_client_source = prisma_client_source
self._sink = sink
self._read_ttl_seconds = read_ttl_seconds
self._readings: LocalTTLCache = InMemoryCache(max_size_in_memory=10_000, default_ttl=read_ttl_seconds)
async def read(self, spend_key: str, budget_duration: str) -> SharedWindow | None:
cached: Final = self._readings.get_cache(key=spend_key)
if isinstance(cached, _Reading):
return cached.window
window: Final = await self._read_row(spend_key=spend_key, budget_duration=budget_duration)
self._readings.set_cache(key=spend_key, value=_Reading(window=window), ttl=self._read_ttl_seconds)
return window
async def open(self, spend_key: str, budget_duration: str, now: float) -> float:
"""When the window every replica shares started, opening one at `now` if none is open.
Falls back to `now`, this replica's own window, when the database cannot answer.
"""
prisma_client: Final = self._prisma_client_source()
if prisma_client is None:
return now
try:
seen: Final = await _fetch_row(
prisma_client=prisma_client, spend_key=spend_key, budget_duration=budget_duration, use_writer=True
)
if seen is not None and seen.is_open(budget_duration=budget_duration, now=now):
return seen.started_at
await open_window_spend_row(
prisma_client=prisma_client,
entity_type=MODEL_BUDGET_WINDOW_ENTITY_TYPE,
entity_id=spend_key,
window_duration=budget_duration,
window_start=datetime.fromtimestamp(now, tz=timezone.utc),
replaces_window_start=None
if seen is None
else datetime.fromtimestamp(seen.started_at, tz=timezone.utc),
)
opened: Final = await _fetch_row(
prisma_client=prisma_client, spend_key=spend_key, budget_duration=budget_duration, use_writer=True
)
except Exception: # noqa: BLE001 # an unreachable database leaves this replica on its own window
verbose_proxy_logger.exception("Unable to open the shared model budget window")
return now
if opened is None or not opened.is_open(budget_duration=budget_duration, now=now):
return now
return opened.started_at
async def add(self, spend_key: str, budget_duration: str, started_at: float, spend: float) -> None:
if self._prisma_client_source() is None:
return
try:
await self._sink(
build_window_spend_transaction(
entity_type=MODEL_BUDGET_WINDOW_ENTITY_TYPE,
entity_id=spend_key,
window_duration=budget_duration,
window_start=datetime.fromtimestamp(started_at, tz=timezone.utc),
spend=spend,
)
)
except Exception: # noqa: BLE001 # spend tracking must never fail the success callback
verbose_proxy_logger.debug("Unable to queue shared model budget spend", exc_info=True)
async def _read_row(self, spend_key: str, budget_duration: str) -> SharedWindow | None:
prisma_client: Final = self._prisma_client_source()
if prisma_client is None:
return None
try:
return await _fetch_row(
prisma_client=prisma_client, spend_key=spend_key, budget_duration=budget_duration, use_writer=False
)
except Exception: # noqa: BLE001 # an unreadable row leaves enforcement on this replica's own counter
verbose_proxy_logger.exception("Unable to read shared model budget spend")
return None
async def _fetch_row(
prisma_client: "PrismaClient", spend_key: str, budget_duration: str, use_writer: bool
) -> SharedWindow | None:
row: Final = await BudgetWindowSpendRepository(prisma_client, use_writer=use_writer).table.find_unique(
where={
"entity_type_entity_id_window_duration": {
"entity_type": MODEL_BUDGET_WINDOW_ENTITY_TYPE,
"entity_id": spend_key,
"window_duration": budget_duration,
}
}
)
if row is None:
return None
window_start: Final = row.window_start
started_at: Final = (
window_start if window_start.tzinfo is not None else window_start.replace(tzinfo=timezone.utc)
).timestamp()
return SharedWindow(started_at=started_at, spend=float(row.spend or 0.0))
PROXY_SHARED_WINDOWS: Final = DatabaseSharedWindows()

View file

@ -1,11 +1,18 @@
import asyncio
import functools
import itertools
import json
import time
from collections.abc import Callable, Iterable, Mapping, Sequence
from collections.abc import Callable, Collection, Iterable, Iterator, Mapping, Sequence
from dataclasses import dataclass
from datetime import datetime, timezone
from types import MappingProxyType
from typing import Final, cast # noqa: TID251 # untyped callback payload requires explicit narrowing
from typing import (
TYPE_CHECKING,
Final,
TypeAlias,
cast, # noqa: TID251 # untyped callback payload requires explicit narrowing
)
import litellm
from litellm._internal_context import with_service_target
@ -16,10 +23,19 @@ from litellm.litellm_core_utils.duration_parser import duration_in_seconds
from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
from litellm.llms.bedrock.common_utils import get_bedrock_base_model
from litellm.proxy._types import Litellm_EntityType, UserAPIKeyAuth
from litellm.proxy.hooks.model_budget_shared_windows import (
PROXY_SHARED_WINDOWS,
LocalTTLCache,
SharedWindow,
SharedWindows,
)
from litellm.router_strategy.budget_limiter import RouterBudgetLimiting, budget_window_seconds
from litellm.router_utils.batch_utils import batch_charged_once_marker_key, batch_id_to_charge_once
from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import BudgetConfig, StandardLoggingPayload
from litellm.types.utils import BudgetConfig, LlmProviders, StandardLoggingPayload
if TYPE_CHECKING:
from litellm.router import Router
VIRTUAL_KEY_SPEND_CACHE_KEY_PREFIX: Final = "virtual_key_spend"
END_USER_SPEND_CACHE_KEY_PREFIX: Final = "end_user_model_spend"
@ -39,10 +55,18 @@ _SPEND_CACHE_KEY_PREFIXES: Final = MappingProxyType(
_LEGACY_REQUEST_MODEL_SCOPES: Final = frozenset({Litellm_EntityType.KEY, Litellm_EntityType.END_USER})
_PER_REPLICA_SCOPES: Final = frozenset({Litellm_EntityType.END_USER})
_PROCESS_STARTED_AT: Final = time.monotonic()
_LATEST_RESET_AT: Final = datetime(9999, 12, 31, tzinfo=timezone.utc).timestamp()
_GLOB_WILDCARD: Final = "*"
ModelAccessGroups: TypeAlias = Callable[[str], Collection[str]]
_PROVIDER_PREFIXES: Final = tuple(f"{provider.value}/" for provider in LlmProviders)
_BUDGET_START_TIME_KEY_PREFIXES: Final = MappingProxyType(
{
Litellm_EntityType.KEY: "virtual_key_budget_start_time",
@ -68,6 +92,15 @@ class ResolvedModelBudget:
budget_config: BudgetConfig
@dataclass(frozen=True, slots=True)
class RoutedModels:
served_names: frozenset[str]
group_members: Mapping[str, frozenset[str]]
def members(self, access_group: str) -> frozenset[str]:
return self.group_members.get(access_group, frozenset())
def team_member_budget_entity_id(user_id: str, team_id: str) -> str:
return f"{len(user_id)}:{user_id}:{team_id}"
@ -131,36 +164,219 @@ def model_budget_start_time_cache_key(
return f"{_BUDGET_START_TIME_KEY_PREFIXES[entity_type]}:{entity_id}:{budget_model}:{budget_duration}"
def resolve_model_budget(model: str, model_max_budget: Mapping[str, object]) -> ResolvedModelBudget | None:
"""Find the `model_max_budget` entry that governs `model`, or None."""
for candidate in _budget_model_candidates(model):
raw_budget_config = model_max_budget.get(candidate)
if raw_budget_config is None:
continue
if (budget_config := _usable_budget_config(raw_budget_config)) is None:
# An entry that will not validate cannot be keyed, so it cannot be
# enforced or incremented. Skip to the next candidate rather than
# raising: raising would abort every other scope's increment and turn
# a config typo into a 500, and stopping here would let one malformed
# specific entry disable a perfectly good bare-family budget beside
# it. The candidate chain already falls through an ABSENT entry, and
# an unparseable one is indistinguishable from absent to enforcement.
# `validate_model_max_budget` rejects these on the write path, so
# reaching here means config.yaml or a direct DB edit.
verbose_proxy_logger.warning(
"Ignoring unusable model_max_budget entry for %s; it cannot be enforced or tracked",
candidate,
)
continue
return ResolvedModelBudget(budget_model=candidate, budget_config=budget_config)
return None
def model_access_groups_of(router: "Router", model: str) -> frozenset[str]:
"""Access groups serving `model`, leaving out any named after a model the router serves.
An entry with that name already budgets that model, so reading it as the group too would quietly widen it.
"""
return frozenset(
group for group in router.get_model_access_groups(model_name=model) if group not in router.model_names
)
def routed_models_of(router: "Router") -> RoutedModels:
served_names: Final = frozenset(router.model_names)
return RoutedModels(
served_names=served_names,
group_members=MappingProxyType(
{
group: frozenset(members)
for group, members in router.get_model_access_groups().items()
if group not in served_names
}
),
)
def proxy_model_access_groups(model: str) -> frozenset[str]:
router: Final = _proxy_router()
return frozenset() if router is None else model_access_groups_of(router=router, model=model)
def proxy_routed_models() -> RoutedModels:
router: Final = _proxy_router()
if router is None:
return RoutedModels(served_names=frozenset(), group_members=MappingProxyType({}))
return routed_models_of(router)
def _proxy_router() -> "Router | None":
try:
from litellm.proxy.proxy_server import llm_router
except ImportError:
return None
return llm_router
def resolve_model_budget(
model: str,
model_max_budget: Mapping[str, object],
model_access_groups: ModelAccessGroups = proxy_model_access_groups,
) -> ResolvedModelBudget | None:
"""Find the `model_max_budget` entry that governs `model`, or None.
An entry named after the model wins, then one named after a model access
group serving it, then the most specific glob (`claude-opus*`). Every model
an access group or glob entry covers draws from that entry's one counter.
"""
resolved: Final = (
_usable_entry(entry=entry, raw_budget_config=model_max_budget[entry])
for entry in _governing_entry_names(
model=model, model_max_budget=model_max_budget, model_access_groups=model_access_groups
)
)
return next((entry for entry in resolved if entry is not None), None)
def _governing_entry_names(
model: str,
model_max_budget: Mapping[str, object],
model_access_groups: ModelAccessGroups,
) -> Iterator[str]:
candidates: Final = _budget_model_candidates(model)
yield from (candidate for candidate in candidates if candidate in model_max_budget)
group_entries: Final = tuple(
sorted(entry for entry in model_max_budget if _GLOB_WILDCARD not in entry and entry not in candidates)
)
if group_entries:
groups: Final = model_access_groups(model)
yield from (entry for entry in group_entries if entry in groups)
yield from sorted(
(entry for entry in model_max_budget if _GLOB_WILDCARD in entry and _glob_matches(entry, candidates)),
key=_glob_precedence,
reverse=True,
)
def _glob_matches(pattern: str, names: Iterable[str]) -> bool:
return any(_glob_matches_name(pattern=pattern, name=name) for name in names)
def _glob_matches_name(pattern: str, name: str) -> bool:
"""Whole-name match where `*` is the only wildcard, in linear time whatever the pattern holds.
Taking each literal piece between two wildcards at its first occurrence is
always safe, since a later occurrence only leaves less room for the rest, so
the match never needs to backtrack the way a `.*` regex does.
"""
head, *middle, tail = pattern.split(_GLOB_WILDCARD)
middle_end: Final = len(name) - len(tail)
if middle_end < len(head) or not name.startswith(head) or not name.endswith(tail):
return False
return functools.reduce(_after_piece(name=name, end=middle_end), middle, len(head)) >= 0
def _after_piece(name: str, end: int) -> Callable[[int, str], int]:
def step(position: int, piece: str) -> int:
found: Final = name.find(piece, position, end) if position >= 0 else -1
return found + len(piece) if found >= 0 else -1
return step
def _glob_precedence(pattern: str) -> tuple[int, int, str]:
"""More literal characters first, then fewer wildcards, so padding a pattern with `*` never makes it win."""
wildcards: Final = pattern.count(_GLOB_WILDCARD)
return len(pattern) - wildcards, -wildcards, pattern
def entries_outranked_by(entry: str, model_max_budget: Collection[str], routed_models: RoutedModels) -> Iterator[str]:
"""Entries that `entry` can take precedence over for some model a request can name.
Two patterns can both match a name the router has not served yet, so a
pattern outranks every less specific pattern. A model or group entry
outranks a pattern that matches one of the names it governs: the model,
the model under any provider prefix or wildcard route, a served name
ending in it, or a member of the group, and every pattern when a member is
itself a wildcard route.
"""
if _GLOB_WILDCARD in entry:
entry_precedence: Final = _glob_precedence(entry)
yield from (
other
for other in model_max_budget
if _GLOB_WILDCARD in other and _glob_precedence(other) < entry_precedence
)
return
entry_members: Final = routed_models.members(entry)
covers_wildcard_route: Final = any(_GLOB_WILDCARD in member for member in entry_members)
governed_candidates: Final = frozenset(
itertools.chain.from_iterable(
_budget_model_candidates(name)
for name in _names_governed_by(entry=entry, entry_members=entry_members, routed_models=routed_models)
)
)
yield from (
other
for other in model_max_budget
if _GLOB_WILDCARD in other and (covers_wildcard_route or _glob_matches(other, governed_candidates))
)
yield from (
other
for other in model_max_budget
if _GLOB_WILDCARD not in other
and _ranks_ahead_of_group(
entry=entry, entry_members=entry_members, group=other, group_members=routed_models.members(other)
)
)
def _names_governed_by(entry: str, entry_members: frozenset[str], routed_models: RoutedModels) -> Iterator[str]:
yield entry
yield from (f"{prefix}{entry}" for prefix in _PROVIDER_PREFIXES)
yield from (
name
for name in routed_models.served_names
if _GLOB_WILDCARD not in name and entry in _budget_model_candidates(name)
)
yield from (name.replace(_GLOB_WILDCARD, entry) for name in routed_models.served_names if _GLOB_WILDCARD in name)
yield from (member for member in entry_members if _GLOB_WILDCARD not in member)
def _ranks_ahead_of_group(entry: str, entry_members: frozenset[str], group: str, group_members: Iterable[str]) -> bool:
return any(
_GLOB_WILDCARD in member
or entry in _budget_model_candidates(member)
or (member in entry_members and entry < group)
for member in group_members
)
def _usable_entry(entry: str, raw_budget_config: object) -> ResolvedModelBudget | None:
if (budget_config := _usable_budget_config(raw_budget_config)) is None:
# An entry that will not validate cannot be keyed, so it cannot be
# enforced or incremented. Skip to the next candidate rather than
# raising: raising would abort every other scope's increment and turn
# a config typo into a 500, and stopping here would let one malformed
# specific entry disable a perfectly good bare-family budget beside
# it. The candidate chain already falls through an ABSENT entry, and
# an unparseable one is indistinguishable from absent to enforcement.
# `validate_model_max_budget` rejects these on the write path, so
# reaching here means config.yaml or a direct DB edit.
verbose_proxy_logger.warning(
"Ignoring unusable model_max_budget entry for %s; it cannot be enforced or tracked",
entry,
)
return None
return ResolvedModelBudget(budget_model=entry, budget_config=budget_config)
def team_model_budget_applies(model: str, key_model_max_budget: Mapping[str, object] | None) -> bool:
"""A key entry that spend-gates `model` overrides the team cap: it is then gated on and billed to the key alone."""
"""A key entry named after `model` that spend-gates it overrides the team cap: it is then gated on and billed to
the key alone. A key pattern or group entry is gated alongside the team cap, never instead of it."""
if not key_model_max_budget:
return True
resolved: Final = resolve_model_budget(model=model, model_max_budget=key_model_max_budget)
resolved: Final = resolve_model_budget(
model=model,
model_max_budget=MappingProxyType(
{
candidate: key_model_max_budget[candidate]
for candidate in _budget_model_candidates(model)
if candidate in key_model_max_budget
}
),
model_access_groups=lambda _: frozenset(),
)
return resolved is None or not _spend_gated(resolved.budget_config)
@ -208,6 +424,7 @@ async def build_model_max_budget_usage(
model_max_budget: Mapping[str, object] | None,
cache: DualCache | None,
now: datetime | None = None,
shared_windows: SharedWindows = PROXY_SHARED_WINDOWS,
) -> dict[str, dict[str, object]]:
"""Current-window spend per configured budget model, as `/key/info` and `/user/info` report it.
@ -231,17 +448,18 @@ async def build_model_max_budget_usage(
)
if not budgets:
return {}
spend_keys: Final = tuple(
model_budget_spend_cache_key(
entity_type=entity_type,
entity_id=entity_id,
budget_model=budget_model,
budget_duration=budget_config.budget_duration,
)
for budget_model, budget_config in budgets
)
windows: Final = await _read_windows(
cache=cache,
spend_keys=tuple(
model_budget_spend_cache_key(
entity_type=entity_type,
entity_id=entity_id,
budget_model=budget_model,
budget_duration=budget_config.budget_duration,
)
for budget_model, budget_config in budgets
),
spend_keys=spend_keys,
start_time_keys=tuple(
model_budget_start_time_cache_key(
entity_type=entity_type,
@ -253,21 +471,57 @@ async def build_model_max_budget_usage(
),
)
current_time: Final = (now or datetime.now(timezone.utc)).timestamp()
shared: Final = tuple(
[
(
await _open_shared_window(
shared_windows=shared_windows,
spend_key=spend_key,
budget_duration=budget_config.budget_duration,
now=current_time,
)
if cache.redis_cache is None
else None
)
for spend_key, (_, budget_config) in zip(spend_keys, budgets, strict=True)
]
)
readings: Final = tuple(
_fuller_reading(local=window, shared=shared_window)
for window, shared_window in zip(windows, shared, strict=True)
)
return {
budget_model: {
"current_spend": round(window.current_spend, 4),
"current_spend": round(reading.current_spend, 4),
"budget_limit": budget_config.max_budget,
"time_period": budget_config.budget_duration,
"budget_reset_at": _window_reset_at(
started_at=window.started_at,
started_at=reading.started_at,
budget_duration=budget_config.budget_duration,
now=current_time,
),
}
for (budget_model, budget_config), window in zip(budgets, windows, strict=True)
for (budget_model, budget_config), reading in zip(budgets, readings, strict=True)
}
async def _open_shared_window(
shared_windows: SharedWindows, spend_key: str, budget_duration: str | None, now: float
) -> SharedWindow | None:
if budget_duration is None:
return None
window: Final = await shared_windows.read(spend_key=spend_key, budget_duration=budget_duration)
if window is None or not window.is_open(budget_duration=budget_duration, now=now):
return None
return window
def _fuller_reading(local: _WindowReading, shared: SharedWindow | None) -> _WindowReading:
if shared is None or shared.spend <= local.current_spend:
return local
return _WindowReading(current_spend=shared.spend, started_at=shared.started_at)
@with_service_target("model_budgets")
async def _read_windows(
cache: DualCache, spend_keys: Sequence[str], start_time_keys: Sequence[str]
@ -337,13 +591,18 @@ def _as_float(value: object) -> float | None:
def _resolve_entity_model_budgets(
model: str,
entity_budgets: Iterable[tuple[Litellm_EntityType, str | None, object]],
model_access_groups: ModelAccessGroups,
) -> tuple[tuple[Litellm_EntityType, str, ResolvedModelBudget], ...]:
"""Drop the scopes that do not budget `model`, keeping only what can be incremented."""
return tuple(
(entity_type, entity_id, resolved)
for entity_type, entity_id, model_max_budget in entity_budgets
if entity_id is not None and isinstance(model_max_budget, Mapping) and model_max_budget
for resolved in (resolve_model_budget(model=model, model_max_budget=model_max_budget),)
for resolved in (
resolve_model_budget(
model=model, model_max_budget=model_max_budget, model_access_groups=model_access_groups
),
)
if resolved is not None and resolved.budget_config.budget_duration is not None
)
@ -367,9 +626,17 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
Example: key=$LITELLM_API_KEY, model=gpt-4o, max_budget=100, time_period=1d
"""
def __init__(self, dual_cache: DualCache, clock: Callable[[], float] = time.time):
def __init__(
self,
dual_cache: DualCache,
clock: Callable[[], float] = time.time,
model_access_groups: ModelAccessGroups = proxy_model_access_groups,
shared_windows: SharedWindows = PROXY_SHARED_WINDOWS,
):
self.dual_cache = dual_cache
self._clock = clock
self._model_access_groups = model_access_groups
self._shared_windows = shared_windows
self.redis_increment_operation_queue = []
self._redis_increment_queue_lock = asyncio.Lock()
self._redis_increment_flush_lock = asyncio.Lock()
@ -511,7 +778,9 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
) -> bool:
if not model_max_budget:
return True
resolved: Final = resolve_model_budget(model=model, model_max_budget=model_max_budget)
resolved: Final = resolve_model_budget(
model=model, model_max_budget=model_max_budget, model_access_groups=self._model_access_groups
)
if resolved is None:
verbose_proxy_logger.debug("Model %s not found in %s model_max_budget", model, entity_type.value)
return True
@ -520,15 +789,26 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
if max_budget is None or max_budget < 0:
return True
current_spend: Final = await self._get_spend_for_model_budget(
local_spend: Final = await self._get_spend_for_model_budget(
entity_type=entity_type,
entity_id=entity_id,
model=model,
resolved=resolved,
)
current_spend: Final = (
local_spend
if local_spend >= max_budget
else await self._with_shared_spend(
entity_type=entity_type, entity_id=entity_id, resolved=resolved, local_spend=local_spend
)
)
if current_spend >= max_budget:
raise litellm.BudgetExceededError(
message=exceeded_message,
message=(
exceeded_message
if resolved.budget_model in _budget_model_candidates(model)
else f"{exceeded_message}, budget_model={resolved.budget_model}"
),
current_cost=current_spend,
max_budget=max_budget,
entity_type=entity_type.value,
@ -566,6 +846,30 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
return current_spend
return current_spend + _as_spend(await self._cached_spend(legacy_spend_key))
async def _with_shared_spend(
self,
entity_type: Litellm_EntityType,
entity_id: str | None,
resolved: ResolvedModelBudget,
local_spend: float,
) -> float:
"""Without Redis each replica's counter holds only its own share, so the window every replica adds to
in the database is the floor."""
if not self._shares_windows_in_database(entity_type):
return local_spend
shared: Final = await _open_shared_window(
shared_windows=self._shared_windows,
spend_key=model_budget_spend_cache_key(
entity_type=entity_type,
entity_id=entity_id,
budget_model=resolved.budget_model,
budget_duration=resolved.budget_config.budget_duration,
),
budget_duration=resolved.budget_config.budget_duration,
now=self._clock(),
)
return local_spend if shared is None else max(local_spend, shared.spend)
async def _cached_spend(self, spend_key: str) -> float | None:
redis_cache: Final = self.dual_cache.redis_cache
if redis_cache is None:
@ -669,7 +973,9 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
),
)
resolved_budgets: Final = _resolve_entity_model_budgets(model=model, entity_budgets=entity_budgets)
resolved_budgets: Final = _resolve_entity_model_budgets(
model=model, entity_budgets=entity_budgets, model_access_groups=self._model_access_groups
)
if not resolved_budgets:
verbose_proxy_logger.debug(
"Not running _PROXY_VirtualKeyModelMaxBudgetLimiter.async_log_success_event: "
@ -723,17 +1029,78 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
ttl_seconds=duration_in_seconds(budget_duration),
):
return
start_time_key: Final = model_budget_start_time_cache_key(
entity_type=entity_type,
entity_id=entity_id,
budget_model=resolved.budget_model,
budget_duration=budget_duration,
)
shares_window: Final = self._shares_windows_in_database(entity_type)
if shares_window:
await self._join_shared_window(
spend_key=spend_key, start_time_key=start_time_key, budget_duration=budget_duration
)
await self._increment_spend_for_key(
budget_config=resolved.budget_config,
spend_key=spend_key,
start_time_key=model_budget_start_time_cache_key(
entity_type=entity_type,
entity_id=entity_id,
budget_model=resolved.budget_model,
budget_duration=budget_duration,
),
start_time_key=start_time_key,
response_cost=response_cost,
)
if shares_window:
await self._share_spend(
spend_key=spend_key,
start_time_key=start_time_key,
budget_duration=budget_duration,
response_cost=response_cost,
)
def _shares_windows_in_database(self, entity_type: Litellm_EntityType) -> bool:
"""End-user ids are whatever the client sends, so their windows stay on each replica rather than adding a
database row per id."""
return self.dual_cache.redis_cache is None and entity_type not in _PER_REPLICA_SCOPES
async def _join_shared_window(self, spend_key: str, start_time_key: str, budget_duration: str) -> None:
"""Without Redis, start this replica's window where the database says every replica's window starts.
Every replica then sends the same window start with its spend, so no flush reads as a newer window that
replaces the spend the others already added.
"""
now: Final = self._clock()
local: Final = self.dual_cache.in_memory_cache
if _open_local_window_start(
local=local, start_time_key=start_time_key, budget_duration=budget_duration, now=now
):
return
started_at: Final = await self._shared_windows.open(
spend_key=spend_key, budget_duration=budget_duration, now=now
)
if _open_local_window_start(
local=local, start_time_key=start_time_key, budget_duration=budget_duration, now=now
):
return
_start_local_window(
local=local,
spend_key=spend_key,
start_time_key=start_time_key,
started_at=started_at,
budget_duration=budget_duration,
now=now,
)
async def _share_spend(
self, spend_key: str, start_time_key: str, budget_duration: str, response_cost: float
) -> None:
now: Final = self._clock()
await self._shared_windows.add(
spend_key=spend_key,
budget_duration=budget_duration,
started_at=await self._get_or_set_budget_start_time(
start_time_key=start_time_key,
current_time=now,
ttl_seconds=budget_window_seconds(now, budget_duration),
),
spend=response_cost,
)
async def _claim_batch_charge(self, spend_key: str, batch_id: str, ttl_seconds: int) -> bool:
marker_key: Final = batch_charged_once_marker_key(spend_key=spend_key, batch_id=batch_id)
@ -743,4 +1110,17 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
return polls == 1
def _open_local_window_start(local: LocalTTLCache, start_time_key: str, budget_duration: str, now: float) -> bool:
started_at: Final = _as_float(local.get_cache(key=start_time_key))
return started_at is not None and now - started_at <= budget_window_seconds(started_at, budget_duration)
def _start_local_window(
local: LocalTTLCache, spend_key: str, start_time_key: str, started_at: float, budget_duration: str, now: float
) -> None:
ttl: Final = started_at + budget_window_seconds(started_at, budget_duration) - now
local.set_cache(key=spend_key, value=0.0, ttl=ttl)
local.set_cache(key=start_time_key, value=started_at, ttl=ttl)
PROXY_VirtualKeyModelMaxBudgetLimiter = _PROXY_VirtualKeyModelMaxBudgetLimiter

View file

@ -139,7 +139,10 @@ from litellm.proxy.common_utils.user_api_key_cache import (
from litellm.proxy.db.db_span import db_span
from litellm.proxy.hooks.key_management_event_hooks import KeyManagementEventHooks
from litellm.proxy.hooks.model_max_budget_limiter import (
RoutedModels,
build_model_max_budget_usage,
entries_outranked_by,
proxy_routed_models,
resolve_model_budget,
)
from litellm.proxy.management.teams.authz import TEAM_OR_ORG_ADMIN, TeamRole, is_team_admin, team_access_denied
@ -1467,6 +1470,7 @@ def _check_team_model_budget_update_authority(
data: UpdateTeamRequest,
user_api_key_dict: UserAPIKeyAuth,
existing_model_max_budget: Mapping[str, object] | None,
routed_models_source: Callable[[], RoutedModels] = proxy_routed_models,
) -> None:
"""Like `_check_team_budget_update_authority`: only a proxy admin may raise, re-window or drop a per-model cap."""
if user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN:
@ -1494,11 +1498,7 @@ def _check_team_model_budget_update_authority(
cap = governing.budget_config
if cap.max_budget is None:
continue
if (
proposed.max_budget is None
or proposed.max_budget > cap.max_budget
or proposed.budget_duration != cap.budget_duration
):
if _looser_model_cap(proposed=proposed, cap=cap):
raise HTTPException(
status_code=403,
detail={
@ -1510,6 +1510,74 @@ def _check_team_model_budget_update_authority(
)
},
)
routed_models: Final = routed_models_source()
for model_name, proposed in requested.items():
_check_outranked_model_caps(
entry=model_name,
proposed=proposed,
existing_model_max_budget=existing_model_max_budget,
routed_models=routed_models,
)
def _check_outranked_model_caps(
entry: str,
proposed: BudgetConfig,
existing_model_max_budget: Mapping[str, object],
routed_models: RoutedModels,
) -> None:
if _keeps_or_tightens(proposed=proposed, existing=_existing_model_cap(existing_model_max_budget.get(entry))):
return
outranked: Final = next(
_looser_than_outranked_caps(entry, proposed, existing_model_max_budget, routed_models), None
)
if outranked is None:
return
outranked_name, cap = outranked
raise HTTPException(
status_code=403,
detail={
"error": (
f"Only a proxy admin can set a team's model_max_budget entry {entry!r} looser than "
f"{outranked_name!r}, which it can take precedence over. Current max_budget={cap.max_budget} per "
f"{cap.budget_duration} (entry {outranked_name!r}), requested={proposed.max_budget} per "
f"{proposed.budget_duration}."
)
},
)
def _looser_than_outranked_caps(
entry: str,
proposed: BudgetConfig,
existing_model_max_budget: Mapping[str, object],
routed_models: RoutedModels,
) -> Iterator[tuple[str, BudgetConfig]]:
outranked_caps: Final = (
(outranked, _existing_model_cap(existing_model_max_budget[outranked]))
for outranked in entries_outranked_by(
entry=entry, model_max_budget=existing_model_max_budget, routed_models=routed_models
)
)
yield from (
(outranked, cap)
for outranked, cap in outranked_caps
if cap is not None and cap.max_budget is not None and _looser_model_cap(proposed=proposed, cap=cap)
)
def _keeps_or_tightens(proposed: BudgetConfig, existing: BudgetConfig | None) -> bool:
return existing is not None and (
existing.max_budget is None or not _looser_model_cap(proposed=proposed, cap=existing)
)
def _looser_model_cap(proposed: BudgetConfig, cap: BudgetConfig) -> bool:
return (
proposed.max_budget is None
or (cap.max_budget is not None and proposed.max_budget > cap.max_budget)
or proposed.budget_duration != cap.budget_duration
)
def _should_auto_add_team_creator(

View file

@ -9,6 +9,7 @@ import pytest
from litellm.proxy.db.budget_window_spend_writer import (
WindowSeedTotals,
commit_window_spend_updates,
open_window_spend_row,
roll_window_spend_row,
spend_logs_seed_totals,
)
@ -405,6 +406,29 @@ async def test_roll_window_spend_row_is_conditional_on_the_stored_window_being_o
assert params[:4] == ("team", "t1", "30d", datetime(2026, 8, 31))
@pytest.mark.asyncio
@pytest.mark.parametrize(
("replaces_window_start", "replaced_param"), [(None, None), (WINDOW_A, datetime(2026, 8, 1))]
)
async def test_open_window_spend_row_moves_only_the_window_the_caller_saw_end(replaces_window_start, replaced_param):
db = _FakeDB()
await open_window_spend_row(
prisma_client=_FakePrismaClient(db),
entity_type="model_max_budget",
entity_id="k1",
window_duration="1d",
window_start=WINDOW_B,
replaces_window_start=replaces_window_start,
)
((query, params),) = db.execute_raw_calls
normalized = " ".join(query.split())
assert "ON CONFLICT (entity_type, entity_id, window_duration) DO UPDATE" in normalized
assert normalized.endswith("WHERE \"LiteLLM_BudgetWindowSpend\".window_start <= ($5::timestamptz AT TIME ZONE 'UTC')")
assert params == ("model_max_budget", "k1", "1d", datetime(2026, 8, 31), replaced_param)
@pytest.mark.asyncio
async def test_seed_receives_the_batch_earliest_start_as_its_cutoff():
db = _FakeDB(existing_rows=[])

View file

@ -0,0 +1,193 @@
from datetime import datetime, timedelta, timezone
from types import SimpleNamespace
from typing import Final
import pytest
from litellm.proxy.db.db_transaction_queue.window_spend_update_queue import WindowSpendTransaction
from litellm.proxy.hooks.model_budget_shared_windows import DatabaseSharedWindows, SharedWindow
SPEND_KEY: Final = "virtual_key_spend:key-hash:claude-opus*:1d"
WINDOW_START: Final = datetime(2026, 10, 20, 9, 30, tzinfo=timezone.utc)
class _Table:
def __init__(self, row: object | None = None, error: Exception | None = None) -> None:
self.row = row
self.error = error
self.reads = 0
async def find_unique(self, where: object) -> object | None:
self.reads = self.reads + 1
if self.error is not None:
raise self.error
return self.row
class _Database:
def __init__(self, table: _Table) -> None:
self.litellm_budgetwindowspend = table
self.opened_with: tuple[object, ...] = ()
async def execute_raw(self, query: str, *params: object) -> int:
self.opened_with = params
self.litellm_budgetwindowspend.row = SimpleNamespace(window_start=params[3], spend=0.0)
return 1
class _OutracedDatabase:
def __init__(self, table: _Table, winner: object) -> None:
self.litellm_budgetwindowspend = table
self._winner = winner
async def execute_raw(self, query: str, *params: object) -> int:
self.litellm_budgetwindowspend.row = self._winner
return 0
class _Sink:
def __init__(self) -> None:
self.transactions: tuple[WindowSpendTransaction, ...] = ()
async def __call__(self, transaction: WindowSpendTransaction) -> None:
self.transactions = (*self.transactions, transaction)
class _LaggingReplica:
def __init__(self, writer: _Database) -> None:
self.litellm_budgetwindowspend = _Table()
self._writer = writer
async def execute_raw(self, query: str, *params: object) -> int:
return await self._writer.execute_raw(query, *params)
def _database_windows(table: _Table | None, sink: _Sink, lagging_replica: bool = False) -> DatabaseSharedWindows:
writer: Final = None if table is None else _Database(table)
prisma: Final = (
None
if writer is None
else SimpleNamespace(db=_LaggingReplica(writer) if lagging_replica else writer, writer_db=writer)
)
return DatabaseSharedWindows(
prisma_client_source=lambda: prisma, # pyright: ignore[reportArgumentType] # duck-typed prisma double
sink=sink,
)
def test_a_window_is_open_until_its_period_has_passed():
window: Final = SharedWindow(started_at=WINDOW_START.timestamp(), spend=1.0)
day: Final = 24 * 60 * 60
assert window.is_open(budget_duration="1d", now=WINDOW_START.timestamp() + day - 1)
assert not window.is_open(budget_duration="1d", now=WINDOW_START.timestamp() + day)
@pytest.mark.asyncio
async def test_read_returns_the_stored_window_and_rereads_only_after_the_cache_expires():
table: Final = _Table(row=SimpleNamespace(window_start=WINDOW_START.replace(tzinfo=None), spend=2.5))
windows: Final = _database_windows(table, _Sink())
first: Final = await windows.read(spend_key=SPEND_KEY, budget_duration="1d")
second: Final = await windows.read(spend_key=SPEND_KEY, budget_duration="1d")
assert first == SharedWindow(started_at=WINDOW_START.timestamp(), spend=2.5)
assert second == first
assert table.reads == 1
@pytest.mark.asyncio
async def test_an_unreadable_table_reads_as_no_shared_window():
table: Final = _Table(error=RuntimeError("database is down"))
windows: Final = _database_windows(table, _Sink())
assert await windows.read(spend_key=SPEND_KEY, budget_duration="1d") is None
@pytest.mark.asyncio
async def test_add_queues_the_charge_for_the_window_it_belongs_to():
sink: Final = _Sink()
windows: Final = _database_windows(_Table(), sink)
await windows.add(spend_key=SPEND_KEY, budget_duration="1d", started_at=WINDOW_START.timestamp(), spend=0.25)
assert len(sink.transactions) == 1
queued: Final = sink.transactions[0]
assert (queued["entity_id"], queued["window_duration"], queued["spend"]) == (SPEND_KEY, "1d", 0.25)
assert datetime.fromisoformat(queued["window_start"]).replace(tzinfo=timezone.utc) == WINDOW_START
@pytest.mark.asyncio
async def test_nothing_is_queued_without_a_database():
sink: Final = _Sink()
windows: Final = _database_windows(None, sink)
await windows.add(spend_key=SPEND_KEY, budget_duration="1d", started_at=WINDOW_START.timestamp(), spend=0.25)
assert sink.transactions == ()
assert await windows.read(spend_key=SPEND_KEY, budget_duration="1d") is None
@pytest.mark.asyncio
async def test_open_joins_the_window_another_replica_already_started():
table: Final = _Table(row=SimpleNamespace(window_start=WINDOW_START.replace(tzinfo=None), spend=2.5))
windows: Final = _database_windows(table, _Sink())
started_at: Final = await windows.open(
spend_key=SPEND_KEY, budget_duration="1d", now=WINDOW_START.timestamp() + 3600
)
assert started_at == WINDOW_START.timestamp()
assert table.row.spend == 2.5
@pytest.mark.asyncio
async def test_open_reads_the_window_from_the_writer_while_the_read_replica_lags():
table: Final = _Table(row=SimpleNamespace(window_start=WINDOW_START.replace(tzinfo=None), spend=2.5))
windows: Final = _database_windows(table, _Sink(), lagging_replica=True)
started_at: Final = await windows.open(
spend_key=SPEND_KEY, budget_duration="1d", now=WINDOW_START.timestamp() + 3600
)
assert started_at == WINDOW_START.timestamp()
@pytest.mark.asyncio
async def test_open_joins_the_window_a_racing_replica_opened_first():
winner: Final = SimpleNamespace(window_start=WINDOW_START.replace(tzinfo=None), spend=2.5)
database: Final = _OutracedDatabase(_Table(), winner)
windows: Final = DatabaseSharedWindows(
prisma_client_source=lambda: SimpleNamespace(db=database, writer_db=database), # pyright: ignore[reportArgumentType] # duck-typed prisma double
sink=_Sink(),
)
started_at: Final = await windows.open(spend_key=SPEND_KEY, budget_duration="1d", now=WINDOW_START.timestamp() + 60)
assert started_at == WINDOW_START.timestamp()
@pytest.mark.asyncio
@pytest.mark.parametrize("stored_start", [None, WINDOW_START - timedelta(days=2)])
async def test_open_starts_a_window_at_now_when_none_is_open(stored_start: datetime | None):
table: Final = _Table(
row=None if stored_start is None else SimpleNamespace(window_start=stored_start.replace(tzinfo=None), spend=9.0)
)
windows: Final = _database_windows(table, _Sink())
started_at: Final = await windows.open(spend_key=SPEND_KEY, budget_duration="1d", now=WINDOW_START.timestamp())
assert started_at == WINDOW_START.timestamp()
assert await windows.read(spend_key=SPEND_KEY, budget_duration="1d") == SharedWindow(
started_at=WINDOW_START.timestamp(), spend=0.0
)
@pytest.mark.asyncio
@pytest.mark.parametrize("table", [None, _Table(error=RuntimeError("database is down"))])
async def test_open_falls_back_to_this_replicas_own_window_without_a_database(table: _Table | None):
windows: Final = _database_windows(table, _Sink())
assert await windows.open(spend_key=SPEND_KEY, budget_duration="1d", now=WINDOW_START.timestamp()) == (
WINDOW_START.timestamp()
)

View file

@ -1,6 +1,9 @@
import asyncio
import functools
import re
import time
from collections.abc import Callable, Mapping
from datetime import datetime, timezone
from types import MappingProxyType
from typing import Final
@ -8,12 +11,19 @@ import pytest
import litellm
from litellm.caching.caching import DualCache
from litellm.proxy._types import Litellm_EntityType
from litellm.proxy._types import Litellm_EntityType, UserAPIKeyAuth
from litellm.proxy.hooks.model_budget_shared_windows import SharedWindow, SharedWindows
from litellm.proxy.hooks.model_max_budget_limiter import (
PROXY_VirtualKeyModelMaxBudgetLimiter,
build_model_max_budget_usage,
model_access_groups_of,
model_budget_spend_cache_key,
resolve_model_budget,
routed_models_of,
team_member_budget_entity_id,
team_model_budget_applies,
)
from litellm.router import Router
from litellm.types.caching import RedisPipelineIncrementOperation
from litellm.types.utils import LiteLLMBatch, Usage
@ -74,10 +84,14 @@ def _event(
}
async def _log_success(
limiter: PROXY_VirtualKeyModelMaxBudgetLimiter, event: dict[str, object], response_obj: object = None
) -> None:
await limiter.async_log_success_event(event, response_obj=response_obj, start_time=None, end_time=None)
async def _poll(limiter: PROXY_VirtualKeyModelMaxBudgetLimiter, batch: LiteLLMBatch, response_cost: float) -> None:
await limiter.async_log_success_event(
_event("aretrieve_batch", response_cost), response_obj=batch, start_time=None, end_time=None
)
await _log_success(limiter, _event("aretrieve_batch", response_cost), response_obj=batch)
async def _chat(
@ -86,7 +100,8 @@ async def _chat(
user_id: str = USER_ID,
team_id: str = TEAM_ID,
) -> None:
await limiter.async_log_success_event(
await _log_success(
limiter,
_event(
"acompletion",
CHAT_COST,
@ -94,9 +109,6 @@ async def _chat(
user_id=user_id,
team_id=team_id,
),
response_obj=None,
start_time=None,
end_time=None,
)
@ -311,3 +323,399 @@ async def test_a_batch_polled_within_every_budget_window_is_never_charged_again(
await _poll(limiter, finished, BATCH_COST)
assert _local_spend(limiter, KEY_SPEND_KEY) == pytest.approx(BATCH_COST)
OPUS_GROUP: Final = "claude-opus"
OPUS_GROUP_MEMBERS: Final = MappingProxyType(
{"claude-opus-4-8": frozenset({OPUS_GROUP}), "claude-opus-5-5": frozenset({OPUS_GROUP})}
)
FAMILY_CAP: Final = {"budget_limit": CHAT_COST, "time_period": "1d"} # mutable-ok: BudgetConfig validates dicts only
ROOMY_CAP: Final = {
"budget_limit": CHAT_COST * 100,
"time_period": "1d",
} # mutable-ok: BudgetConfig validates dicts only
WINDOW_EPOCH: Final = datetime(2026, 10, 20, 9, 30, tzinfo=timezone.utc).timestamp()
def _model_access_groups(groups_by_model: Mapping[str, frozenset[str]]) -> Callable[[str], frozenset[str]]:
return lambda model: groups_by_model.get(model, frozenset())
class _SharedWindowsDouble:
def __init__(self) -> None:
self.rows: Mapping[str, SharedWindow] = MappingProxyType({})
self.queued: tuple[tuple[str, float, float], ...] = ()
async def read(self, spend_key: str, budget_duration: str) -> SharedWindow | None:
return self.rows.get(spend_key)
async def open(self, spend_key: str, budget_duration: str, now: float) -> float:
await asyncio.sleep(0)
stored: Final = self.rows.get(spend_key)
if stored is not None and stored.is_open(budget_duration=budget_duration, now=now):
return stored.started_at
self.rows = MappingProxyType({**self.rows, spend_key: SharedWindow(started_at=now, spend=0.0)})
return now
async def add(self, spend_key: str, budget_duration: str, started_at: float, spend: float) -> None:
self.queued = (*self.queued, (spend_key, started_at, spend))
def flush(self) -> None:
self.rows = functools.reduce(_upsert_window_spend, self.queued, self.rows)
self.queued = ()
def _upsert_window_spend(
rows: Mapping[str, SharedWindow], queued: tuple[str, float, float]
) -> Mapping[str, SharedWindow]:
spend_key, started_at, spend = queued
stored: Final = rows.get(spend_key)
row: Final = (
SharedWindow(started_at=stored.started_at, spend=stored.spend + spend)
if stored is not None and stored.started_at >= started_at
else SharedWindow(started_at=started_at, spend=spend)
)
return MappingProxyType({**rows, spend_key: row})
def _family_limiter(
groups_by_model: Mapping[str, frozenset[str]] = OPUS_GROUP_MEMBERS,
dual_cache: DualCache | None = None,
shared_windows: SharedWindows | None = None,
clock: Callable[[], float] = time.time,
) -> PROXY_VirtualKeyModelMaxBudgetLimiter:
return PROXY_VirtualKeyModelMaxBudgetLimiter(
dual_cache=dual_cache or DualCache(),
clock=clock,
model_access_groups=_model_access_groups(groups_by_model),
shared_windows=shared_windows or _SharedWindowsDouble(),
)
def _key(model_max_budget: Mapping[str, object]) -> UserAPIKeyAuth:
return UserAPIKeyAuth(token=KEY_HASH, key_alias="opus-family", model_max_budget=dict(model_max_budget))
async def _spend_on(
limiter: PROXY_VirtualKeyModelMaxBudgetLimiter, model: str, model_max_budget: Mapping[str, object]
) -> None:
await _log_success(
limiter,
{
"call_type": "acompletion",
"standard_logging_object": {
"call_type": "acompletion",
"response_cost": CHAT_COST,
"model": f"anthropic/{model}",
"model_group": model,
"metadata": {"user_api_key_hash": KEY_HASH},
},
"litellm_params": {"metadata": {"user_api_key_model_max_budget": dict(model_max_budget)}},
},
)
@pytest.mark.asyncio
@pytest.mark.parametrize("family_entry", ["claude-opus*", OPUS_GROUP])
async def test_every_opus_version_draws_from_one_family_entry(family_entry: str):
budget: Final = {family_entry: FAMILY_CAP}
limiter: Final = _family_limiter()
assert await limiter.is_key_within_model_budget(user_api_key_dict=_key(budget), model="claude-opus-5-5")
await _spend_on(limiter, model="claude-opus-4-8", model_max_budget=budget)
with pytest.raises(
litellm.BudgetExceededError, match=rf"model=claude-opus-5-5, budget_model={re.escape(family_entry)}$"
):
await limiter.is_key_within_model_budget(user_api_key_dict=_key(budget), model="claude-opus-5-5")
assert await limiter.is_key_within_model_budget(user_api_key_dict=_key(budget), model="claude-sonnet-5")
@pytest.mark.asyncio
async def test_a_model_added_to_the_access_group_later_is_covered_with_no_key_change():
budget: Final = {OPUS_GROUP: FAMILY_CAP}
shared_cache: Final = DualCache()
before_group_change: Final = _family_limiter(dual_cache=shared_cache)
after_group_change: Final = _family_limiter(
groups_by_model={**OPUS_GROUP_MEMBERS, "claude-opus-6": frozenset({OPUS_GROUP})}, dual_cache=shared_cache
)
await _spend_on(before_group_change, model="claude-opus-4-8", model_max_budget=budget)
assert await before_group_change.is_key_within_model_budget(user_api_key_dict=_key(budget), model="claude-opus-6")
with pytest.raises(litellm.BudgetExceededError):
await after_group_change.is_key_within_model_budget(user_api_key_dict=_key(budget), model="claude-opus-6")
@pytest.mark.asyncio
@pytest.mark.parametrize(
("budget", "checked_model", "blocked"),
[
({"claude-opus-4-8": ROOMY_CAP, "claude-opus*": FAMILY_CAP}, "claude-opus-4-8", False),
({"claude-opus-4-8": ROOMY_CAP, "claude-opus*": FAMILY_CAP}, "claude-opus-5-5", False),
({OPUS_GROUP: FAMILY_CAP, "claude-opus*": ROOMY_CAP}, "claude-opus-5-5", True),
({"claude-opus-4*": FAMILY_CAP, "claude-opus*": ROOMY_CAP}, "claude-opus-4-7", True),
({"claude-opus-4*": FAMILY_CAP, "claude-opus*": ROOMY_CAP}, "claude-opus-5-5", False),
({"claude-opus-4*": {"budget_limit": CHAT_COST}, "claude-opus*": FAMILY_CAP}, "claude-opus-5-5", True),
({"claude-opus-4*": FAMILY_CAP, "c" + "*" * 15: ROOMY_CAP}, "claude-opus-4-7", True),
],
)
async def test_the_most_specific_entry_takes_the_spend_and_gates_the_request(
budget: Mapping[str, object], checked_model: str, blocked: bool
):
limiter: Final = _family_limiter()
await _spend_on(limiter, model="claude-opus-4-8", model_max_budget=budget)
if blocked:
with pytest.raises(litellm.BudgetExceededError):
await limiter.is_key_within_model_budget(user_api_key_dict=_key(budget), model=checked_model)
else:
assert await limiter.is_key_within_model_budget(user_api_key_dict=_key(budget), model=checked_model)
@pytest.mark.asyncio
async def test_an_exact_entry_refusal_reads_as_before_for_a_provider_prefixed_model():
budget: Final = {"claude-opus-4-8": FAMILY_CAP}
limiter: Final = _family_limiter()
await _spend_on(limiter, model="claude-opus-4-8", model_max_budget=budget)
with pytest.raises(litellm.BudgetExceededError, match=r"model=anthropic/claude-opus-4-8$"):
await limiter.is_key_within_model_budget(user_api_key_dict=_key(budget), model="anthropic/claude-opus-4-8")
@pytest.mark.parametrize(
("key_entry", "team_cap_applies"),
[("claude-opus-4-8", False), ("anthropic/claude-opus-4-8", True), ("claude-opus*", True), (OPUS_GROUP, True)],
)
def test_only_a_key_entry_named_after_the_model_replaces_the_team_cap(key_entry: str, team_cap_applies: bool):
assert team_model_budget_applies(model="claude-opus-4-8", key_model_max_budget={key_entry: ROOMY_CAP}) is (
team_cap_applies
)
@pytest.mark.asyncio
async def test_end_user_windows_stay_on_the_replica_without_redis():
budget: Final = {"claude-opus*": FAMILY_CAP}
table: Final = _SharedWindowsDouble()
limiter: Final = _family_limiter(shared_windows=table)
await _log_success(
limiter,
{
"call_type": "acompletion",
"standard_logging_object": {
"call_type": "acompletion",
"response_cost": CHAT_COST,
"model": "anthropic/claude-opus-4-8",
"model_group": "claude-opus-4-8",
"end_user": "end-user-1",
"metadata": {},
},
"litellm_params": {"metadata": {"user_api_key_end_user_model_max_budget": budget}},
},
)
table.flush()
assert dict(table.rows) == {}
with pytest.raises(litellm.BudgetExceededError, match="model=claude-opus-5-5"):
await limiter.is_end_user_within_model_budget(
end_user_id="end-user-1", end_user_model_max_budget=budget, model="claude-opus-5-5"
)
def test_an_access_group_named_after_a_served_model_never_widens_that_models_entry():
router: Final = Router(
model_list=[
{
"model_name": "gpt-4o",
"litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-test"},
"model_info": {"access_groups": ["gpt-4o", "openai-models"]},
},
{
"model_name": "gpt-4o-mini",
"litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-test"},
"model_info": {"access_groups": ["gpt-4o", "openai-models"]},
},
]
)
def governing_entry(model_max_budget: Mapping[str, object]) -> str | None:
resolved: Final = resolve_model_budget(
model="gpt-4o-mini",
model_max_budget=model_max_budget,
model_access_groups=lambda model: model_access_groups_of(router=router, model=model),
)
return None if resolved is None else resolved.budget_model
assert governing_entry({"gpt-4o": FAMILY_CAP}) is None
assert governing_entry({"openai-models": FAMILY_CAP}) == "openai-models"
assert routed_models_of(router).members("gpt-4o") == frozenset()
assert routed_models_of(router).members("openai-models") == frozenset({"gpt-4o", "gpt-4o-mini"})
@pytest.mark.asyncio
async def test_key_info_usage_reports_the_family_total():
budget: Final = {"claude-opus*": ROOMY_CAP}
limiter: Final = _family_limiter()
await _spend_on(limiter, model="claude-opus-4-8", model_max_budget=budget)
await _spend_on(limiter, model="claude-opus-5-5", model_max_budget=budget)
usage: Final = await build_model_max_budget_usage(
entity_type=Litellm_EntityType.KEY,
entity_id=KEY_HASH,
model_max_budget=budget,
cache=limiter.dual_cache,
)
assert usage["claude-opus*"]["current_spend"] == round(CHAT_COST * 2, 4)
@pytest.mark.parametrize(
("pattern", "model", "governs"),
[
("claude-opus*", "claude-opus-4-8", True),
("claude-opus*", "anthropic/claude-opus-4-8", True),
("anthropic/claude-opus*", "anthropic/claude-opus-4-8", True),
("anthropic/claude-opus*", "claude-opus-4-8", False),
("claude-opus*", "my-claude-opus-4-8", False),
("claude-opus*", "claude-sonnet-5", False),
("gpt-4.1*", "gpt-401-mini", False),
("*", "claude-sonnet-5", True),
],
)
def test_glob_entries_match_the_whole_model_name_with_only_star_as_a_wildcard(pattern: str, model: str, governs: bool):
resolved: Final = resolve_model_budget(
model=model, model_max_budget={pattern: FAMILY_CAP}, model_access_groups=_model_access_groups({})
)
assert (resolved is not None and resolved.budget_model == pattern) is governs
@pytest.mark.asyncio
async def test_replicas_without_redis_enforce_one_family_cap_together():
budget: Final = {"claude-opus*": FAMILY_CAP}
table: Final = _SharedWindowsDouble()
replica_a: Final = _family_limiter(shared_windows=table)
replica_b: Final = _family_limiter(shared_windows=table)
await _spend_on(replica_a, model="claude-opus-4-8", model_max_budget=budget)
table.flush()
with pytest.raises(litellm.BudgetExceededError, match="model=claude-opus-5-5"):
await replica_b.is_key_within_model_budget(user_api_key_dict=_key(budget), model="claude-opus-5-5")
@pytest.mark.asyncio
async def test_replicas_sharing_redis_enforce_the_family_cap_without_the_database():
budget: Final = {"claude-opus*": FAMILY_CAP}
redis: Final = _SharedRedisDouble()
table: Final = _SharedWindowsDouble()
replica_a: Final = _family_limiter(
dual_cache=DualCache(redis_cache=redis), # pyright: ignore[reportArgumentType] # duck-typed Redis double
shared_windows=table,
)
replica_b: Final = _family_limiter(
dual_cache=DualCache(redis_cache=redis), # pyright: ignore[reportArgumentType] # duck-typed Redis double
shared_windows=table,
)
await _spend_on(replica_a, model="claude-opus-4-8", model_max_budget=budget)
await _drain_redis_pushes()
table.flush()
with pytest.raises(litellm.BudgetExceededError, match="model=claude-opus-5-5"):
await replica_b.is_key_within_model_budget(user_api_key_dict=_key(budget), model="claude-opus-5-5")
assert dict(table.rows) == {}
@pytest.mark.asyncio
async def test_a_replica_charges_the_open_shared_window_instead_of_restarting_it():
budget: Final = {"claude-opus*": {"budget_limit": CHAT_COST * 1.5, "time_period": "1d"}}
table: Final = _SharedWindowsDouble()
clock: Final = _Clock()
clock.advance(WINDOW_EPOCH)
replica_a: Final = _family_limiter(shared_windows=table, clock=clock.now)
replica_b: Final = _family_limiter(shared_windows=table, clock=clock.now)
await _spend_on(replica_a, model="claude-opus-4-8", model_max_budget=budget)
table.flush()
clock.advance(3600)
await _spend_on(replica_b, model="claude-opus-5-5", model_max_budget=budget)
table.flush()
for replica in (replica_a, replica_b):
with pytest.raises(litellm.BudgetExceededError):
await replica.is_key_within_model_budget(user_api_key_dict=_key(budget), model="claude-opus-5-5")
@pytest.mark.asyncio
async def test_key_info_on_one_replica_reports_spend_from_every_replica():
budget: Final = {"claude-opus*": ROOMY_CAP}
table: Final = _SharedWindowsDouble()
replica_a: Final = _family_limiter(shared_windows=table)
replica_b: Final = _family_limiter(shared_windows=table)
await _spend_on(replica_a, model="claude-opus-4-8", model_max_budget=budget)
await _spend_on(replica_b, model="claude-opus-5-5", model_max_budget=budget)
table.flush()
usage: Final = await build_model_max_budget_usage(
entity_type=Litellm_EntityType.KEY,
entity_id=KEY_HASH,
model_max_budget=budget,
cache=replica_a.dual_cache,
shared_windows=table,
)
assert usage["claude-opus*"]["current_spend"] == round(CHAT_COST * 2, 4)
assert usage["claude-opus*"]["budget_reset_at"] is not None
@pytest.mark.asyncio
@pytest.mark.parametrize("budget_model", ["claude-opus*", "claude-opus-4-8"])
async def test_replicas_starting_a_window_moments_apart_keep_each_others_spend(budget_model: str):
budget: Final = {budget_model: ROOMY_CAP}
table: Final = _SharedWindowsDouble()
clock: Final = _Clock()
clock.advance(WINDOW_EPOCH)
replica_a: Final = _family_limiter(shared_windows=table, clock=clock.now)
replica_b: Final = _family_limiter(shared_windows=table, clock=clock.now)
await _spend_on(replica_a, model="claude-opus-4-8", model_max_budget=budget)
clock.advance(1)
await _spend_on(replica_b, model="claude-opus-4-8", model_max_budget=budget)
table.flush()
usage: Final = await build_model_max_budget_usage(
entity_type=Litellm_EntityType.KEY,
entity_id=KEY_HASH,
model_max_budget=budget,
cache=DualCache(),
now=datetime.fromtimestamp(clock.now(), tz=timezone.utc),
shared_windows=table,
)
assert usage[budget_model]["current_spend"] == round(CHAT_COST * 2, 4)
@pytest.mark.asyncio
async def test_concurrent_first_charges_on_one_replica_both_count():
budget: Final = {"claude-opus*": ROOMY_CAP}
clock: Final = _Clock()
clock.advance(WINDOW_EPOCH)
limiter: Final = _family_limiter(shared_windows=_SharedWindowsDouble(), clock=clock.now)
await asyncio.gather(
_spend_on(limiter, model="claude-opus-4-8", model_max_budget=budget),
_spend_on(limiter, model="claude-opus-5-5", model_max_budget=budget),
)
usage: Final = await build_model_max_budget_usage(
entity_type=Litellm_EntityType.KEY,
entity_id=KEY_HASH,
model_max_budget=budget,
cache=limiter.dual_cache,
now=datetime.fromtimestamp(clock.now(), tz=timezone.utc),
shared_windows=_SharedWindowsDouble(),
)
assert usage["claude-opus*"]["current_spend"] == round(CHAT_COST * 2, 4)

View file

@ -16580,6 +16580,7 @@ async def test_info_key_fn_includes_model_max_budget_usage(monkeypatch):
}
mock_prisma_client = AsyncMock()
mock_prisma_client.db.litellm_budgetwindowspend.find_unique = AsyncMock(return_value=None)
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client)
mock_user_api_key_cache = AsyncMock()
monkeypatch.setattr(
@ -16703,6 +16704,7 @@ async def test_info_key_fn_v2_includes_model_max_budget_usage(monkeypatch):
model_max_budget = {"gpt-4o": {"budget_limit": 1.00, "time_period": "7d"}}
mock_prisma_client = AsyncMock()
mock_prisma_client.db.litellm_budgetwindowspend.find_unique = AsyncMock(return_value=None)
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client)
mock_user_api_key_cache = AsyncMock()
monkeypatch.setattr(
@ -16767,6 +16769,7 @@ async def test_info_key_fn_budget_table_fallback(monkeypatch):
}
mock_prisma_client = AsyncMock()
mock_prisma_client.db.litellm_budgetwindowspend.find_unique = AsyncMock(return_value=None)
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client)
mock_user_api_key_cache = AsyncMock()
monkeypatch.setattr(
@ -16840,6 +16843,7 @@ async def test_info_key_fn_v2_budget_table_fallback(monkeypatch):
}
mock_prisma_client = AsyncMock()
mock_prisma_client.db.litellm_budgetwindowspend.find_unique = AsyncMock(return_value=None)
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client)
mock_user_api_key_cache = AsyncMock()
monkeypatch.setattr(
@ -17215,6 +17219,7 @@ async def test_info_key_fn_reads_the_configured_budget_model_key(monkeypatch):
}
mock_prisma_client = AsyncMock()
mock_prisma_client.db.litellm_budgetwindowspend.find_unique = AsyncMock(return_value=None)
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client)
mock_user_api_key_cache = AsyncMock()
monkeypatch.setattr(

View file

@ -44,6 +44,7 @@ from litellm.proxy._types import (
UpdateTeamRequest,
UserAPIKeyAuth, # Import UserAPIKeyAuth
)
from litellm.proxy.hooks.model_max_budget_limiter import RoutedModels
from litellm.proxy.management.teams.authz import TeamAccess
from litellm.proxy.management_endpoints.team_admin_field_permissions import (
PROXY_CONTROLLED_TEAM_METADATA_KEYS,
@ -17593,6 +17594,130 @@ def test_team_admin_can_tighten_or_keep_team_model_caps(requested) -> None:
)
_TEAM_OPUS_FAMILY_CAP: Final = MappingProxyType({"claude-opus*": {"max_budget": 10.0, "budget_duration": "1d"}})
_TEAM_OPUS_GROUP_CAP: Final = MappingProxyType({"z-opus-group": {"max_budget": 10.0, "budget_duration": "1d"}})
_TEAM_ROUTED_MODELS: Final = RoutedModels(
served_names=frozenset(
{"claude-opus-4-8", "claude-opus-5-5", "gemini-3-pro", "gpt-4o", "my-team/gpt-4o", "team-route/*"}
),
group_members=MappingProxyType(
{
"opus-group": frozenset({"claude-opus-4-8"}),
"z-opus-group": frozenset({"claude-opus-4-8"}),
"a-opus-group": frozenset({"claude-opus-4-8", "claude-opus-5-5"}),
"zz-opus-group": frozenset({"claude-opus-4-8"}),
"a-gemini-group": frozenset({"gemini-3-pro"}),
"z-anthropic-group": frozenset({"anthropic/*"}),
}
),
)
def _check_team_admin_model_caps(existing: Mapping[str, object], requested: Mapping[str, object]) -> None:
from litellm.proxy.management_endpoints.team_endpoints import _check_team_model_budget_update_authority
_check_team_model_budget_update_authority(
data=UpdateTeamRequest(team_id="t1", model_max_budget=dict(requested)),
user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, user_id="team-admin"),
existing_model_max_budget=existing,
routed_models_source=lambda: _TEAM_ROUTED_MODELS,
)
@pytest.mark.parametrize(
("existing", "added"),
[
(_TEAM_OPUS_FAMILY_CAP, {"opus-group": {"max_budget": 1000.0, "budget_duration": "1d"}}),
(_TEAM_OPUS_FAMILY_CAP, {"opus-group": {"max_budget": 10.0, "budget_duration": "7d"}}),
(_TEAM_OPUS_FAMILY_CAP, {"*laude-opus-4-8": {"max_budget": 1000.0, "budget_duration": "1d"}}),
(_TEAM_OPUS_GROUP_CAP, {"a-opus-group": {"max_budget": 1000.0, "budget_duration": "1d"}}),
(_TEAM_OPUS_GROUP_CAP, {"claude-opus-4-8": {"max_budget": 1000.0, "budget_duration": "1d"}}),
(
{"z-anthropic-group": {"max_budget": 10.0, "budget_duration": "1d"}},
{"claude-opus-4-8": {"max_budget": 1000.0, "budget_duration": "1d"}},
),
(_TEAM_OPUS_FAMILY_CAP, {"claude-opus-4-8": {"max_budget": 1000.0, "budget_duration": "1d"}}),
(
{"anthropic/*": {"max_budget": 10.0, "budget_duration": "1d"}},
{"claude-opus-4-8": {"max_budget": 1000.0, "budget_duration": "1d"}},
),
(
{"my-team/*": {"max_budget": 10.0, "budget_duration": "1d"}},
{"gpt-4o": {"max_budget": 1000.0, "budget_duration": "1d"}},
),
(
{"team-route/*": {"max_budget": 10.0, "budget_duration": "1d"}},
{"gpt-4o": {"max_budget": 1000.0, "budget_duration": "1d"}},
),
],
ids=[
"group_over_pattern",
"group_rewindows_pattern",
"more_specific_pattern",
"earlier_group_sharing_a_model",
"model_inside_group",
"model_under_wildcard_group",
"model_matching_pattern",
"model_under_provider_pattern",
"model_under_served_prefix",
"model_under_wildcard_route",
],
)
def test_team_admin_cannot_add_an_entry_looser_than_a_cap_it_outranks(
existing: Mapping[str, object], added: Mapping[str, object]
) -> None:
with pytest.raises(HTTPException) as exc:
_check_team_admin_model_caps(existing=existing, requested={**existing, **added})
assert exc.value.status_code == 403
assert "proxy admin" in exc.value.detail["error"]
assert repr(next(iter(existing))) in exc.value.detail["error"]
@pytest.mark.parametrize(
("existing", "requested"),
[
(_TEAM_OPUS_FAMILY_CAP, {**_TEAM_OPUS_FAMILY_CAP, "opus-group": {"max_budget": 5.0, "budget_duration": "1d"}}),
(_TEAM_OPUS_FAMILY_CAP, {**_TEAM_OPUS_FAMILY_CAP, "claude*": {"max_budget": 1000.0, "budget_duration": "1d"}}),
(_TEAM_OPUS_GROUP_CAP, {**_TEAM_OPUS_GROUP_CAP, "a-gemini-group": {"max_budget": 1000.0, "budget_duration": "1d"}}),
(_TEAM_OPUS_GROUP_CAP, {**_TEAM_OPUS_GROUP_CAP, "zz-opus-group": {"max_budget": 1000.0, "budget_duration": "1d"}}),
(_TEAM_OPUS_GROUP_CAP, {**_TEAM_OPUS_GROUP_CAP, "gpt-4o": {"max_budget": 1000.0, "budget_duration": "1d"}}),
(_TEAM_OPUS_FAMILY_CAP, {**_TEAM_OPUS_FAMILY_CAP, "gpt-4o": {"max_budget": 1000.0, "budget_duration": "1d"}}),
(
_TEAM_OPUS_FAMILY_CAP,
{**_TEAM_OPUS_FAMILY_CAP, "a-gemini-group": {"max_budget": 1000.0, "budget_duration": "1d"}},
),
(
{**_TEAM_OPUS_FAMILY_CAP, "opus-group": {"max_budget": 100.0, "budget_duration": "1d"}},
{**_TEAM_OPUS_FAMILY_CAP, "opus-group": {"max_budget": 100.0, "budget_duration": "1d"}},
),
(
{**_TEAM_OPUS_FAMILY_CAP, "opus-group": {"max_budget": 100.0, "budget_duration": "1d"}},
{**_TEAM_OPUS_FAMILY_CAP, "opus-group": {"max_budget": 50.0, "budget_duration": "1d"}},
),
(
{**_TEAM_OPUS_FAMILY_CAP, "opus-group": {"rpm_limit": 10}},
{**_TEAM_OPUS_FAMILY_CAP, "opus-group": {"rpm_limit": 20}},
),
],
ids=[
"tighter_group",
"less_specific_pattern",
"group_without_shared_models",
"later_group_sharing_a_model",
"model_outside_group",
"model_outside_pattern",
"group_outside_pattern",
"unchanged_looser_entry",
"tightened_looser_entry",
"uncapped_entry_edited",
],
)
def test_team_admin_can_add_entries_that_never_lift_an_outranked_cap(
existing: Mapping[str, object], requested: Mapping[str, object]
) -> None:
assert _check_team_admin_model_caps(existing=existing, requested=requested) is None
def test_team_model_cap_authority_skips_omitted_field_malformed_rows_and_proxy_admins() -> None:
from litellm.proxy.management_endpoints.team_endpoints import _check_team_model_budget_update_authority

View file

@ -74,6 +74,7 @@ _IN_MEMORY_ONLY_CALLERS: Final = frozenset(
"litellm/proxy/container_endpoints/ownership.py",
"litellm/proxy/discovery_endpoints/agent_skills_endpoints.py",
"litellm/proxy/guardrails/guardrail_hooks/straiker/straiker.py",
"litellm/proxy/hooks/model_budget_shared_windows.py",
"litellm/proxy/spend_tracking/key_metadata_recovery.py",
"litellm/proxy/ui_crud_endpoints/latest_release_endpoints.py",
"litellm/responses/litellm_completion_transformation/transformation.py",

View file

@ -1,5 +1,6 @@
import userEvent from "@testing-library/user-event";
import { describe, expect, it, vi } from "vitest";
import { renderWithProviders, screen } from "../../../tests/test-utils";
import { fireEvent, renderWithProviders, screen } from "../../../tests/test-utils";
import { MODEL_MAX_BUDGET_PREMIUM_HINT, ModelMaxBudgetEditor, type ModelMaxBudget } from "./ModelMaxBudgetEditor";
const STORED: ModelMaxBudget = { "gpt-4o": { budget_limit: 5, time_period: "30d" } };
@ -67,3 +68,65 @@ describe("ModelMaxBudgetEditor with an enterprise license", () => {
expect(addButton()).toBeEnabled();
});
});
describe("ModelMaxBudgetEditor model families", () => {
it("stores a typed pattern that matches no listed model as one budget entry", async () => {
const user = userEvent.setup();
const onChange = vi.fn();
renderWithProviders(
<ModelMaxBudgetEditor
value={{}}
onChange={onChange}
availableModels={["gpt-4o", "claude-opus-4-8"]}
premiumUser={true}
/>,
);
await user.click(addButton());
const modelField = screen.getByPlaceholderText("Select model");
await user.type(modelField, "claude-opus*");
await user.click(await screen.findByRole("option", { name: /claude-opus\*/ }));
fireEvent.change(screen.getByPlaceholderText("Max spend ($)"), { target: { value: "10" } });
expect(onChange).toHaveBeenLastCalledWith({ "claude-opus*": { budget_limit: 10, time_period: "30d" } });
});
it("does not offer a typed pattern that another row already budgets", async () => {
const user = userEvent.setup();
renderWithProviders(
<ModelMaxBudgetEditor
value={{ "claude-opus*": { budget_limit: 10, time_period: "1d" } }}
onChange={vi.fn()}
availableModels={["gpt-4o", "claude-opus-4-8"]}
premiumUser={true}
/>,
);
await user.click(addButton());
const [, newRowModelField] = screen.getAllByPlaceholderText("Select model");
await user.type(newRowModelField, "claude-opus*");
expect(screen.queryByRole("option", { name: /claude-opus\*/ })).not.toBeInTheDocument();
await user.clear(newRowModelField);
await user.type(newRowModelField, "claude-sonnet*");
expect(await screen.findByRole("option", { name: /claude-sonnet\*/ })).toBeInTheDocument();
});
it("does not offer a typed name that is neither listed nor a pattern", async () => {
const user = userEvent.setup();
renderWithProviders(
<ModelMaxBudgetEditor
value={{}}
onChange={vi.fn()}
availableModels={["gpt-4o", "claude-opus-4-8"]}
premiumUser={true}
/>,
);
await user.click(addButton());
await user.type(screen.getByPlaceholderText("Select model"), "claude-opsu-4-8");
expect(screen.queryByRole("option", { name: /claude-opsu-4-8/ })).not.toBeInTheDocument();
expect(screen.getByText("No models found")).toBeInTheDocument();
});
});

View file

@ -50,6 +50,8 @@ export const MODEL_BUDGET_PERIOD_OPTIONS = [
const DEFAULT_PERIOD = "30d";
const MODEL_PATTERN_VALUE = { hint: "Model pattern", accepts: (typed: string) => typed.includes("*") };
export const entriesToModelMaxBudget = (entries: readonly ModelBudgetEntry[]): ModelMaxBudget =>
Object.fromEntries(
entries
@ -121,7 +123,7 @@ export function ModelMaxBudgetEditor({
const blurb = (
<div className="text-xs text-muted-foreground">
{premiumUser
? "Cap spend per model over its own window. A budget set on the bare model name also covers the provider-prefixed spelling of that model."
? "Cap spend per model over its own window. A budget set on the bare model name also covers the provider-prefixed spelling of that model. Type an access group or a pattern like claude-opus* to share one budget across every model it covers."
: MODEL_MAX_BUDGET_PREMIUM_HINT}
</div>
);
@ -165,6 +167,8 @@ export function ModelMaxBudgetEditor({
onValueChange={(model) => updateEntry(entry.id, { model })}
placeholder="Select model"
emptyText="No models found"
customValue={MODEL_PATTERN_VALUE}
reservedValues={entries.flatMap((other) => (other.id !== entry.id && other.model ? [other.model] : []))}
disabled={!premiumUser}
/>
</div>

View file

@ -63,6 +63,24 @@ describe("SearchSelect", () => {
expect(onValueChange).toHaveBeenLastCalledWith("team-3");
});
it("offers the typed text as a value only when the caller accepts it", async () => {
const user = userEvent.setup();
const onValueChange = vi.fn();
const customValue = { hint: "Custom value", accepts: (typed: string) => typed.endsWith("*") };
const { rerender } = render(<SearchSelect options={OPTIONS} onValueChange={onValueChange} />);
const input = screen.getByRole("combobox");
await user.click(input);
fireEvent.change(input, { target: { value: "team-*" } });
expect(screen.queryByRole("option", { name: /team-\*/ })).not.toBeInTheDocument();
rerender(<SearchSelect options={OPTIONS} onValueChange={onValueChange} customValue={customValue} />);
fireEvent.change(input, { target: { value: "team-4" } });
expect(screen.queryByRole("option", { name: /team-4/ })).not.toBeInTheDocument();
fireEvent.change(input, { target: { value: "team-*" } });
await user.click(await screen.findByRole("option", { name: /team-\*/ }));
expect(onValueChange).toHaveBeenLastCalledWith("team-*");
});
it("filters the options client-side as you type", async () => {
const user = userEvent.setup();
render(<SearchSelect options={OPTIONS} onValueChange={vi.fn()} />);

View file

@ -1,6 +1,6 @@
"use client";
import React from "react";
import React, { useState } from "react";
import {
Combobox,
ComboboxContent,
@ -19,6 +19,11 @@ export interface SearchSelectOption {
icon?: React.ReactNode;
}
interface CustomValue {
hint: string;
accepts: (typed: string) => boolean;
}
interface SearchSelectProps {
options: SearchSelectOption[];
value?: string | null;
@ -29,6 +34,8 @@ interface SearchSelectProps {
className?: string;
inputId?: string;
allowClear?: boolean;
customValue?: CustomValue;
reservedValues?: readonly string[];
"aria-label"?: string;
id?: string;
}
@ -49,13 +56,22 @@ export function SearchSelect({
className,
inputId,
allowClear = true,
customValue,
reservedValues = [],
"aria-label": ariaLabel,
id,
}: SearchSelectProps) {
const [query, setQuery] = useState("");
const selected =
value == null || value === "" ? null : options.find((option) => option.value === value) ?? { label: value, value };
const items =
const known =
selected !== null && !options.some((option) => option.value === selected.value) ? [selected, ...options] : options;
const typed = query.trim();
const isNewValue = typed !== "" && !reservedValues.includes(typed) && !known.some((option) => option.value === typed);
const items =
customValue !== undefined && isNewValue && customValue.accepts(typed)
? [...known, { label: typed, value: typed, sublabel: customValue.hint }]
: known;
return (
<Combobox
@ -65,6 +81,7 @@ export function SearchSelect({
isItemEqualToValue={(a: SearchSelectOption, b: SearchSelectOption) => a.value === b.value}
itemToStringLabel={(item: SearchSelectOption) => item.label}
filter={matchesQuery}
onInputValueChange={setQuery}
disabled={disabled}
>
<ComboboxInput