diff --git a/litellm/integrations/otel/model/spans.py b/litellm/integrations/otel/model/spans.py
index d8be607150c..59519bf3428 100644
--- a/litellm/integrations/otel/model/spans.py
+++ b/litellm/integrations/otel/model/spans.py
@@ -434,6 +434,7 @@ _POSTGRES_OPERATION_BY_CALL_TYPE: Final[Mapping[str, PostgresOperation]] = Mappi
"reset_budget_windows": PostgresOperation("select", None),
"write_budget_windows": PostgresOperation("update", None),
"roll_window_spend_row": PostgresOperation("update", None),
+ "open_window_spend_row": PostgresOperation("upsert", None),
"seed_window_spend": PostgresOperation("select", None),
"select_window_spend_rows": PostgresOperation("select", None),
"commit_window_spend_updates": PostgresOperation("upsert", None),
diff --git a/litellm/proxy/db/budget_window_spend_writer.py b/litellm/proxy/db/budget_window_spend_writer.py
index 82c95cfe998..99b49331932 100644
--- a/litellm/proxy/db/budget_window_spend_writer.py
+++ b/litellm/proxy/db/budget_window_spend_writer.py
@@ -64,6 +64,16 @@ _ROLL_WINDOW_SPEND_SQL: Final = (
"AND window_start < ($4::timestamptz AT TIME ZONE 'UTC')"
)
+_OPEN_WINDOW_SPEND_SQL: Final = (
+ 'INSERT INTO "LiteLLM_BudgetWindowSpend" '
+ "(entity_type, entity_id, window_duration, window_start, spend, created_at, updated_at) "
+ "VALUES ($1, $2, $3, ($4::timestamptz AT TIME ZONE 'UTC'), 0, "
+ "($4::timestamptz AT TIME ZONE 'UTC'), ($4::timestamptz AT TIME ZONE 'UTC')) "
+ "ON CONFLICT (entity_type, entity_id, window_duration) DO UPDATE SET "
+ "window_start = EXCLUDED.window_start, spend = 0, updated_at = EXCLUDED.updated_at "
+ "WHERE \"LiteLLM_BudgetWindowSpend\".window_start <= ($5::timestamptz AT TIME ZONE 'UTC')"
+)
+
_SEED_FROM_SPEND_LOGS_KEY_SQL: Final = (
"SELECT COALESCE(SUM(spend), 0.0) AS total, "
"COALESCE(SUM(spend) FILTER (WHERE \"startTime\" < ($3::timestamptz AT TIME ZONE 'UTC')), 0.0) AS before_batch "
@@ -389,3 +399,29 @@ async def roll_window_spend_row(
to_naive_utc(new_window_start),
to_naive_utc(datetime.now(timezone.utc)),
)
+
+
+async def open_window_spend_row(
+ prisma_client: "PrismaClient",
+ entity_type: str,
+ entity_id: str,
+ window_duration: str,
+ window_start: datetime,
+ replaces_window_start: datetime | None,
+) -> None:
+ """Start a window at `window_start` unless another writer already started one.
+
+ The row is created when it does not exist, and moved onto the new window only
+ while it still holds `replaces_window_start`, the ended window the caller saw.
+ A writer that lost the race leaves the winner's window in place, so reading
+ the row back gives every writer the same window start.
+ """
+ async with db_span("open_window_spend_row", "LiteLLM_BudgetWindowSpend"):
+ await prisma_client.db.execute_raw(
+ _OPEN_WINDOW_SPEND_SQL,
+ entity_type,
+ entity_id,
+ window_duration,
+ to_naive_utc(window_start),
+ None if replaces_window_start is None else to_naive_utc(replaces_window_start),
+ )
diff --git a/litellm/proxy/hooks/model_budget_shared_windows.py b/litellm/proxy/hooks/model_budget_shared_windows.py
new file mode 100644
index 00000000000..21f44ae6d6d
--- /dev/null
+++ b/litellm/proxy/hooks/model_budget_shared_windows.py
@@ -0,0 +1,173 @@
+from collections.abc import Awaitable, Callable
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from typing import TYPE_CHECKING, Final, Protocol, TypeAlias
+
+from litellm._logging import verbose_proxy_logger
+from litellm.caching.in_memory_cache import InMemoryCache
+from litellm.proxy.db.budget_window_spend_writer import open_window_spend_row
+from litellm.proxy.db.db_transaction_queue.window_spend_update_queue import (
+ WindowSpendTransaction,
+ build_window_spend_transaction,
+)
+from litellm.repositories.table_repositories import BudgetWindowSpendRepository
+from litellm.router_strategy.budget_limiter import budget_window_seconds
+
+if TYPE_CHECKING:
+ from litellm.proxy.utils import PrismaClient
+
+MODEL_BUDGET_WINDOW_ENTITY_TYPE: Final = "model_max_budget"
+_READ_TTL_SECONDS: Final = 5
+
+
+@dataclass(frozen=True, slots=True)
+class SharedWindow:
+ started_at: float
+ spend: float
+
+ def is_open(self, budget_duration: str, now: float) -> bool:
+ return now < self.started_at + budget_window_seconds(self.started_at, budget_duration)
+
+
+class LocalTTLCache(Protocol):
+ def get_cache(self, key: str) -> object: ...
+
+ def set_cache(self, key: str, value: object, *, ttl: float) -> None: ...
+
+
+class SharedWindows(Protocol):
+ async def read(self, spend_key: str, budget_duration: str) -> SharedWindow | None: ...
+
+ async def open(self, spend_key: str, budget_duration: str, now: float) -> float: ...
+
+ async def add(self, spend_key: str, budget_duration: str, started_at: float, spend: float) -> None: ...
+
+
+PrismaClientSource: TypeAlias = Callable[[], "PrismaClient | None"]
+WindowSpendSink: TypeAlias = Callable[[WindowSpendTransaction], Awaitable[None]]
+
+
+def _proxy_prisma_client() -> "PrismaClient | None":
+ try:
+ from litellm.proxy.proxy_server import prisma_client
+ except ImportError:
+ return None
+ return prisma_client
+
+
+async def _queue_on_proxy_spend_writer(transaction: WindowSpendTransaction) -> None:
+ from litellm.proxy.proxy_server import proxy_logging_obj
+
+ await proxy_logging_obj.db_spend_update_writer.window_spend_update_queue.add_update(transaction)
+
+
+@dataclass(frozen=True, slots=True)
+class _Reading:
+ window: SharedWindow | None
+
+
+class DatabaseSharedWindows:
+ def __init__(
+ self,
+ prisma_client_source: PrismaClientSource = _proxy_prisma_client,
+ sink: WindowSpendSink = _queue_on_proxy_spend_writer,
+ read_ttl_seconds: int = _READ_TTL_SECONDS,
+ ) -> None:
+ self._prisma_client_source = prisma_client_source
+ self._sink = sink
+ self._read_ttl_seconds = read_ttl_seconds
+ self._readings: LocalTTLCache = InMemoryCache(max_size_in_memory=10_000, default_ttl=read_ttl_seconds)
+
+ async def read(self, spend_key: str, budget_duration: str) -> SharedWindow | None:
+ cached: Final = self._readings.get_cache(key=spend_key)
+ if isinstance(cached, _Reading):
+ return cached.window
+ window: Final = await self._read_row(spend_key=spend_key, budget_duration=budget_duration)
+ self._readings.set_cache(key=spend_key, value=_Reading(window=window), ttl=self._read_ttl_seconds)
+ return window
+
+ async def open(self, spend_key: str, budget_duration: str, now: float) -> float:
+ """When the window every replica shares started, opening one at `now` if none is open.
+
+ Falls back to `now`, this replica's own window, when the database cannot answer.
+ """
+ prisma_client: Final = self._prisma_client_source()
+ if prisma_client is None:
+ return now
+ try:
+ seen: Final = await _fetch_row(
+ prisma_client=prisma_client, spend_key=spend_key, budget_duration=budget_duration, use_writer=True
+ )
+ if seen is not None and seen.is_open(budget_duration=budget_duration, now=now):
+ return seen.started_at
+ await open_window_spend_row(
+ prisma_client=prisma_client,
+ entity_type=MODEL_BUDGET_WINDOW_ENTITY_TYPE,
+ entity_id=spend_key,
+ window_duration=budget_duration,
+ window_start=datetime.fromtimestamp(now, tz=timezone.utc),
+ replaces_window_start=None
+ if seen is None
+ else datetime.fromtimestamp(seen.started_at, tz=timezone.utc),
+ )
+ opened: Final = await _fetch_row(
+ prisma_client=prisma_client, spend_key=spend_key, budget_duration=budget_duration, use_writer=True
+ )
+ except Exception: # noqa: BLE001 # an unreachable database leaves this replica on its own window
+ verbose_proxy_logger.exception("Unable to open the shared model budget window")
+ return now
+ if opened is None or not opened.is_open(budget_duration=budget_duration, now=now):
+ return now
+ return opened.started_at
+
+ async def add(self, spend_key: str, budget_duration: str, started_at: float, spend: float) -> None:
+ if self._prisma_client_source() is None:
+ return
+ try:
+ await self._sink(
+ build_window_spend_transaction(
+ entity_type=MODEL_BUDGET_WINDOW_ENTITY_TYPE,
+ entity_id=spend_key,
+ window_duration=budget_duration,
+ window_start=datetime.fromtimestamp(started_at, tz=timezone.utc),
+ spend=spend,
+ )
+ )
+ except Exception: # noqa: BLE001 # spend tracking must never fail the success callback
+ verbose_proxy_logger.debug("Unable to queue shared model budget spend", exc_info=True)
+
+ async def _read_row(self, spend_key: str, budget_duration: str) -> SharedWindow | None:
+ prisma_client: Final = self._prisma_client_source()
+ if prisma_client is None:
+ return None
+ try:
+ return await _fetch_row(
+ prisma_client=prisma_client, spend_key=spend_key, budget_duration=budget_duration, use_writer=False
+ )
+ except Exception: # noqa: BLE001 # an unreadable row leaves enforcement on this replica's own counter
+ verbose_proxy_logger.exception("Unable to read shared model budget spend")
+ return None
+
+
+async def _fetch_row(
+ prisma_client: "PrismaClient", spend_key: str, budget_duration: str, use_writer: bool
+) -> SharedWindow | None:
+ row: Final = await BudgetWindowSpendRepository(prisma_client, use_writer=use_writer).table.find_unique(
+ where={
+ "entity_type_entity_id_window_duration": {
+ "entity_type": MODEL_BUDGET_WINDOW_ENTITY_TYPE,
+ "entity_id": spend_key,
+ "window_duration": budget_duration,
+ }
+ }
+ )
+ if row is None:
+ return None
+ window_start: Final = row.window_start
+ started_at: Final = (
+ window_start if window_start.tzinfo is not None else window_start.replace(tzinfo=timezone.utc)
+ ).timestamp()
+ return SharedWindow(started_at=started_at, spend=float(row.spend or 0.0))
+
+
+PROXY_SHARED_WINDOWS: Final = DatabaseSharedWindows()
diff --git a/litellm/proxy/hooks/model_max_budget_limiter.py b/litellm/proxy/hooks/model_max_budget_limiter.py
index bc9cee1cb1d..c704a9ab7d6 100644
--- a/litellm/proxy/hooks/model_max_budget_limiter.py
+++ b/litellm/proxy/hooks/model_max_budget_limiter.py
@@ -1,11 +1,18 @@
import asyncio
+import functools
+import itertools
import json
import time
-from collections.abc import Callable, Iterable, Mapping, Sequence
+from collections.abc import Callable, Collection, Iterable, Iterator, Mapping, Sequence
from dataclasses import dataclass
from datetime import datetime, timezone
from types import MappingProxyType
-from typing import Final, cast # noqa: TID251 # untyped callback payload requires explicit narrowing
+from typing import (
+ TYPE_CHECKING,
+ Final,
+ TypeAlias,
+ cast, # noqa: TID251 # untyped callback payload requires explicit narrowing
+)
import litellm
from litellm._internal_context import with_service_target
@@ -16,10 +23,19 @@ from litellm.litellm_core_utils.duration_parser import duration_in_seconds
from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
from litellm.llms.bedrock.common_utils import get_bedrock_base_model
from litellm.proxy._types import Litellm_EntityType, UserAPIKeyAuth
+from litellm.proxy.hooks.model_budget_shared_windows import (
+ PROXY_SHARED_WINDOWS,
+ LocalTTLCache,
+ SharedWindow,
+ SharedWindows,
+)
from litellm.router_strategy.budget_limiter import RouterBudgetLimiting, budget_window_seconds
from litellm.router_utils.batch_utils import batch_charged_once_marker_key, batch_id_to_charge_once
from litellm.types.llms.openai import AllMessageValues
-from litellm.types.utils import BudgetConfig, StandardLoggingPayload
+from litellm.types.utils import BudgetConfig, LlmProviders, StandardLoggingPayload
+
+if TYPE_CHECKING:
+ from litellm.router import Router
VIRTUAL_KEY_SPEND_CACHE_KEY_PREFIX: Final = "virtual_key_spend"
END_USER_SPEND_CACHE_KEY_PREFIX: Final = "end_user_model_spend"
@@ -39,10 +55,18 @@ _SPEND_CACHE_KEY_PREFIXES: Final = MappingProxyType(
_LEGACY_REQUEST_MODEL_SCOPES: Final = frozenset({Litellm_EntityType.KEY, Litellm_EntityType.END_USER})
+_PER_REPLICA_SCOPES: Final = frozenset({Litellm_EntityType.END_USER})
+
_PROCESS_STARTED_AT: Final = time.monotonic()
_LATEST_RESET_AT: Final = datetime(9999, 12, 31, tzinfo=timezone.utc).timestamp()
+_GLOB_WILDCARD: Final = "*"
+
+ModelAccessGroups: TypeAlias = Callable[[str], Collection[str]]
+
+_PROVIDER_PREFIXES: Final = tuple(f"{provider.value}/" for provider in LlmProviders)
+
_BUDGET_START_TIME_KEY_PREFIXES: Final = MappingProxyType(
{
Litellm_EntityType.KEY: "virtual_key_budget_start_time",
@@ -68,6 +92,15 @@ class ResolvedModelBudget:
budget_config: BudgetConfig
+@dataclass(frozen=True, slots=True)
+class RoutedModels:
+ served_names: frozenset[str]
+ group_members: Mapping[str, frozenset[str]]
+
+ def members(self, access_group: str) -> frozenset[str]:
+ return self.group_members.get(access_group, frozenset())
+
+
def team_member_budget_entity_id(user_id: str, team_id: str) -> str:
return f"{len(user_id)}:{user_id}:{team_id}"
@@ -131,36 +164,219 @@ def model_budget_start_time_cache_key(
return f"{_BUDGET_START_TIME_KEY_PREFIXES[entity_type]}:{entity_id}:{budget_model}:{budget_duration}"
-def resolve_model_budget(model: str, model_max_budget: Mapping[str, object]) -> ResolvedModelBudget | None:
- """Find the `model_max_budget` entry that governs `model`, or None."""
- for candidate in _budget_model_candidates(model):
- raw_budget_config = model_max_budget.get(candidate)
- if raw_budget_config is None:
- continue
- if (budget_config := _usable_budget_config(raw_budget_config)) is None:
- # An entry that will not validate cannot be keyed, so it cannot be
- # enforced or incremented. Skip to the next candidate rather than
- # raising: raising would abort every other scope's increment and turn
- # a config typo into a 500, and stopping here would let one malformed
- # specific entry disable a perfectly good bare-family budget beside
- # it. The candidate chain already falls through an ABSENT entry, and
- # an unparseable one is indistinguishable from absent to enforcement.
- # `validate_model_max_budget` rejects these on the write path, so
- # reaching here means config.yaml or a direct DB edit.
- verbose_proxy_logger.warning(
- "Ignoring unusable model_max_budget entry for %s; it cannot be enforced or tracked",
- candidate,
- )
- continue
- return ResolvedModelBudget(budget_model=candidate, budget_config=budget_config)
- return None
+def model_access_groups_of(router: "Router", model: str) -> frozenset[str]:
+ """Access groups serving `model`, leaving out any named after a model the router serves.
+
+ An entry with that name already budgets that model, so reading it as the group too would quietly widen it.
+ """
+ return frozenset(
+ group for group in router.get_model_access_groups(model_name=model) if group not in router.model_names
+ )
+
+
+def routed_models_of(router: "Router") -> RoutedModels:
+ served_names: Final = frozenset(router.model_names)
+ return RoutedModels(
+ served_names=served_names,
+ group_members=MappingProxyType(
+ {
+ group: frozenset(members)
+ for group, members in router.get_model_access_groups().items()
+ if group not in served_names
+ }
+ ),
+ )
+
+
+def proxy_model_access_groups(model: str) -> frozenset[str]:
+ router: Final = _proxy_router()
+ return frozenset() if router is None else model_access_groups_of(router=router, model=model)
+
+
+def proxy_routed_models() -> RoutedModels:
+ router: Final = _proxy_router()
+ if router is None:
+ return RoutedModels(served_names=frozenset(), group_members=MappingProxyType({}))
+ return routed_models_of(router)
+
+
+def _proxy_router() -> "Router | None":
+ try:
+ from litellm.proxy.proxy_server import llm_router
+ except ImportError:
+ return None
+ return llm_router
+
+
+def resolve_model_budget(
+ model: str,
+ model_max_budget: Mapping[str, object],
+ model_access_groups: ModelAccessGroups = proxy_model_access_groups,
+) -> ResolvedModelBudget | None:
+ """Find the `model_max_budget` entry that governs `model`, or None.
+
+ An entry named after the model wins, then one named after a model access
+ group serving it, then the most specific glob (`claude-opus*`). Every model
+ an access group or glob entry covers draws from that entry's one counter.
+ """
+ resolved: Final = (
+ _usable_entry(entry=entry, raw_budget_config=model_max_budget[entry])
+ for entry in _governing_entry_names(
+ model=model, model_max_budget=model_max_budget, model_access_groups=model_access_groups
+ )
+ )
+ return next((entry for entry in resolved if entry is not None), None)
+
+
+def _governing_entry_names(
+ model: str,
+ model_max_budget: Mapping[str, object],
+ model_access_groups: ModelAccessGroups,
+) -> Iterator[str]:
+ candidates: Final = _budget_model_candidates(model)
+ yield from (candidate for candidate in candidates if candidate in model_max_budget)
+ group_entries: Final = tuple(
+ sorted(entry for entry in model_max_budget if _GLOB_WILDCARD not in entry and entry not in candidates)
+ )
+ if group_entries:
+ groups: Final = model_access_groups(model)
+ yield from (entry for entry in group_entries if entry in groups)
+ yield from sorted(
+ (entry for entry in model_max_budget if _GLOB_WILDCARD in entry and _glob_matches(entry, candidates)),
+ key=_glob_precedence,
+ reverse=True,
+ )
+
+
+def _glob_matches(pattern: str, names: Iterable[str]) -> bool:
+ return any(_glob_matches_name(pattern=pattern, name=name) for name in names)
+
+
+def _glob_matches_name(pattern: str, name: str) -> bool:
+ """Whole-name match where `*` is the only wildcard, in linear time whatever the pattern holds.
+
+ Taking each literal piece between two wildcards at its first occurrence is
+ always safe, since a later occurrence only leaves less room for the rest, so
+ the match never needs to backtrack the way a `.*` regex does.
+ """
+ head, *middle, tail = pattern.split(_GLOB_WILDCARD)
+ middle_end: Final = len(name) - len(tail)
+ if middle_end < len(head) or not name.startswith(head) or not name.endswith(tail):
+ return False
+ return functools.reduce(_after_piece(name=name, end=middle_end), middle, len(head)) >= 0
+
+
+def _after_piece(name: str, end: int) -> Callable[[int, str], int]:
+ def step(position: int, piece: str) -> int:
+ found: Final = name.find(piece, position, end) if position >= 0 else -1
+ return found + len(piece) if found >= 0 else -1
+
+ return step
+
+
+def _glob_precedence(pattern: str) -> tuple[int, int, str]:
+ """More literal characters first, then fewer wildcards, so padding a pattern with `*` never makes it win."""
+ wildcards: Final = pattern.count(_GLOB_WILDCARD)
+ return len(pattern) - wildcards, -wildcards, pattern
+
+
+def entries_outranked_by(entry: str, model_max_budget: Collection[str], routed_models: RoutedModels) -> Iterator[str]:
+ """Entries that `entry` can take precedence over for some model a request can name.
+
+ Two patterns can both match a name the router has not served yet, so a
+ pattern outranks every less specific pattern. A model or group entry
+ outranks a pattern that matches one of the names it governs: the model,
+ the model under any provider prefix or wildcard route, a served name
+ ending in it, or a member of the group, and every pattern when a member is
+ itself a wildcard route.
+ """
+ if _GLOB_WILDCARD in entry:
+ entry_precedence: Final = _glob_precedence(entry)
+ yield from (
+ other
+ for other in model_max_budget
+ if _GLOB_WILDCARD in other and _glob_precedence(other) < entry_precedence
+ )
+ return
+ entry_members: Final = routed_models.members(entry)
+ covers_wildcard_route: Final = any(_GLOB_WILDCARD in member for member in entry_members)
+ governed_candidates: Final = frozenset(
+ itertools.chain.from_iterable(
+ _budget_model_candidates(name)
+ for name in _names_governed_by(entry=entry, entry_members=entry_members, routed_models=routed_models)
+ )
+ )
+ yield from (
+ other
+ for other in model_max_budget
+ if _GLOB_WILDCARD in other and (covers_wildcard_route or _glob_matches(other, governed_candidates))
+ )
+ yield from (
+ other
+ for other in model_max_budget
+ if _GLOB_WILDCARD not in other
+ and _ranks_ahead_of_group(
+ entry=entry, entry_members=entry_members, group=other, group_members=routed_models.members(other)
+ )
+ )
+
+
+def _names_governed_by(entry: str, entry_members: frozenset[str], routed_models: RoutedModels) -> Iterator[str]:
+ yield entry
+ yield from (f"{prefix}{entry}" for prefix in _PROVIDER_PREFIXES)
+ yield from (
+ name
+ for name in routed_models.served_names
+ if _GLOB_WILDCARD not in name and entry in _budget_model_candidates(name)
+ )
+ yield from (name.replace(_GLOB_WILDCARD, entry) for name in routed_models.served_names if _GLOB_WILDCARD in name)
+ yield from (member for member in entry_members if _GLOB_WILDCARD not in member)
+
+
+def _ranks_ahead_of_group(entry: str, entry_members: frozenset[str], group: str, group_members: Iterable[str]) -> bool:
+ return any(
+ _GLOB_WILDCARD in member
+ or entry in _budget_model_candidates(member)
+ or (member in entry_members and entry < group)
+ for member in group_members
+ )
+
+
+def _usable_entry(entry: str, raw_budget_config: object) -> ResolvedModelBudget | None:
+ if (budget_config := _usable_budget_config(raw_budget_config)) is None:
+ # An entry that will not validate cannot be keyed, so it cannot be
+ # enforced or incremented. Skip to the next candidate rather than
+ # raising: raising would abort every other scope's increment and turn
+ # a config typo into a 500, and stopping here would let one malformed
+ # specific entry disable a perfectly good bare-family budget beside
+ # it. The candidate chain already falls through an ABSENT entry, and
+ # an unparseable one is indistinguishable from absent to enforcement.
+ # `validate_model_max_budget` rejects these on the write path, so
+ # reaching here means config.yaml or a direct DB edit.
+ verbose_proxy_logger.warning(
+ "Ignoring unusable model_max_budget entry for %s; it cannot be enforced or tracked",
+ entry,
+ )
+ return None
+ return ResolvedModelBudget(budget_model=entry, budget_config=budget_config)
def team_model_budget_applies(model: str, key_model_max_budget: Mapping[str, object] | None) -> bool:
- """A key entry that spend-gates `model` overrides the team cap: it is then gated on and billed to the key alone."""
+ """A key entry named after `model` that spend-gates it overrides the team cap: it is then gated on and billed to
+ the key alone. A key pattern or group entry is gated alongside the team cap, never instead of it."""
if not key_model_max_budget:
return True
- resolved: Final = resolve_model_budget(model=model, model_max_budget=key_model_max_budget)
+ resolved: Final = resolve_model_budget(
+ model=model,
+ model_max_budget=MappingProxyType(
+ {
+ candidate: key_model_max_budget[candidate]
+ for candidate in _budget_model_candidates(model)
+ if candidate in key_model_max_budget
+ }
+ ),
+ model_access_groups=lambda _: frozenset(),
+ )
return resolved is None or not _spend_gated(resolved.budget_config)
@@ -208,6 +424,7 @@ async def build_model_max_budget_usage(
model_max_budget: Mapping[str, object] | None,
cache: DualCache | None,
now: datetime | None = None,
+ shared_windows: SharedWindows = PROXY_SHARED_WINDOWS,
) -> dict[str, dict[str, object]]:
"""Current-window spend per configured budget model, as `/key/info` and `/user/info` report it.
@@ -231,17 +448,18 @@ async def build_model_max_budget_usage(
)
if not budgets:
return {}
+ spend_keys: Final = tuple(
+ model_budget_spend_cache_key(
+ entity_type=entity_type,
+ entity_id=entity_id,
+ budget_model=budget_model,
+ budget_duration=budget_config.budget_duration,
+ )
+ for budget_model, budget_config in budgets
+ )
windows: Final = await _read_windows(
cache=cache,
- spend_keys=tuple(
- model_budget_spend_cache_key(
- entity_type=entity_type,
- entity_id=entity_id,
- budget_model=budget_model,
- budget_duration=budget_config.budget_duration,
- )
- for budget_model, budget_config in budgets
- ),
+ spend_keys=spend_keys,
start_time_keys=tuple(
model_budget_start_time_cache_key(
entity_type=entity_type,
@@ -253,21 +471,57 @@ async def build_model_max_budget_usage(
),
)
current_time: Final = (now or datetime.now(timezone.utc)).timestamp()
+ shared: Final = tuple(
+ [
+ (
+ await _open_shared_window(
+ shared_windows=shared_windows,
+ spend_key=spend_key,
+ budget_duration=budget_config.budget_duration,
+ now=current_time,
+ )
+ if cache.redis_cache is None
+ else None
+ )
+ for spend_key, (_, budget_config) in zip(spend_keys, budgets, strict=True)
+ ]
+ )
+ readings: Final = tuple(
+ _fuller_reading(local=window, shared=shared_window)
+ for window, shared_window in zip(windows, shared, strict=True)
+ )
return {
budget_model: {
- "current_spend": round(window.current_spend, 4),
+ "current_spend": round(reading.current_spend, 4),
"budget_limit": budget_config.max_budget,
"time_period": budget_config.budget_duration,
"budget_reset_at": _window_reset_at(
- started_at=window.started_at,
+ started_at=reading.started_at,
budget_duration=budget_config.budget_duration,
now=current_time,
),
}
- for (budget_model, budget_config), window in zip(budgets, windows, strict=True)
+ for (budget_model, budget_config), reading in zip(budgets, readings, strict=True)
}
+async def _open_shared_window(
+ shared_windows: SharedWindows, spend_key: str, budget_duration: str | None, now: float
+) -> SharedWindow | None:
+ if budget_duration is None:
+ return None
+ window: Final = await shared_windows.read(spend_key=spend_key, budget_duration=budget_duration)
+ if window is None or not window.is_open(budget_duration=budget_duration, now=now):
+ return None
+ return window
+
+
+def _fuller_reading(local: _WindowReading, shared: SharedWindow | None) -> _WindowReading:
+ if shared is None or shared.spend <= local.current_spend:
+ return local
+ return _WindowReading(current_spend=shared.spend, started_at=shared.started_at)
+
+
@with_service_target("model_budgets")
async def _read_windows(
cache: DualCache, spend_keys: Sequence[str], start_time_keys: Sequence[str]
@@ -337,13 +591,18 @@ def _as_float(value: object) -> float | None:
def _resolve_entity_model_budgets(
model: str,
entity_budgets: Iterable[tuple[Litellm_EntityType, str | None, object]],
+ model_access_groups: ModelAccessGroups,
) -> tuple[tuple[Litellm_EntityType, str, ResolvedModelBudget], ...]:
"""Drop the scopes that do not budget `model`, keeping only what can be incremented."""
return tuple(
(entity_type, entity_id, resolved)
for entity_type, entity_id, model_max_budget in entity_budgets
if entity_id is not None and isinstance(model_max_budget, Mapping) and model_max_budget
- for resolved in (resolve_model_budget(model=model, model_max_budget=model_max_budget),)
+ for resolved in (
+ resolve_model_budget(
+ model=model, model_max_budget=model_max_budget, model_access_groups=model_access_groups
+ ),
+ )
if resolved is not None and resolved.budget_config.budget_duration is not None
)
@@ -367,9 +626,17 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
Example: key=$LITELLM_API_KEY, model=gpt-4o, max_budget=100, time_period=1d
"""
- def __init__(self, dual_cache: DualCache, clock: Callable[[], float] = time.time):
+ def __init__(
+ self,
+ dual_cache: DualCache,
+ clock: Callable[[], float] = time.time,
+ model_access_groups: ModelAccessGroups = proxy_model_access_groups,
+ shared_windows: SharedWindows = PROXY_SHARED_WINDOWS,
+ ):
self.dual_cache = dual_cache
self._clock = clock
+ self._model_access_groups = model_access_groups
+ self._shared_windows = shared_windows
self.redis_increment_operation_queue = []
self._redis_increment_queue_lock = asyncio.Lock()
self._redis_increment_flush_lock = asyncio.Lock()
@@ -511,7 +778,9 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
) -> bool:
if not model_max_budget:
return True
- resolved: Final = resolve_model_budget(model=model, model_max_budget=model_max_budget)
+ resolved: Final = resolve_model_budget(
+ model=model, model_max_budget=model_max_budget, model_access_groups=self._model_access_groups
+ )
if resolved is None:
verbose_proxy_logger.debug("Model %s not found in %s model_max_budget", model, entity_type.value)
return True
@@ -520,15 +789,26 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
if max_budget is None or max_budget < 0:
return True
- current_spend: Final = await self._get_spend_for_model_budget(
+ local_spend: Final = await self._get_spend_for_model_budget(
entity_type=entity_type,
entity_id=entity_id,
model=model,
resolved=resolved,
)
+ current_spend: Final = (
+ local_spend
+ if local_spend >= max_budget
+ else await self._with_shared_spend(
+ entity_type=entity_type, entity_id=entity_id, resolved=resolved, local_spend=local_spend
+ )
+ )
if current_spend >= max_budget:
raise litellm.BudgetExceededError(
- message=exceeded_message,
+ message=(
+ exceeded_message
+ if resolved.budget_model in _budget_model_candidates(model)
+ else f"{exceeded_message}, budget_model={resolved.budget_model}"
+ ),
current_cost=current_spend,
max_budget=max_budget,
entity_type=entity_type.value,
@@ -566,6 +846,30 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
return current_spend
return current_spend + _as_spend(await self._cached_spend(legacy_spend_key))
+ async def _with_shared_spend(
+ self,
+ entity_type: Litellm_EntityType,
+ entity_id: str | None,
+ resolved: ResolvedModelBudget,
+ local_spend: float,
+ ) -> float:
+ """Without Redis each replica's counter holds only its own share, so the window every replica adds to
+ in the database is the floor."""
+ if not self._shares_windows_in_database(entity_type):
+ return local_spend
+ shared: Final = await _open_shared_window(
+ shared_windows=self._shared_windows,
+ spend_key=model_budget_spend_cache_key(
+ entity_type=entity_type,
+ entity_id=entity_id,
+ budget_model=resolved.budget_model,
+ budget_duration=resolved.budget_config.budget_duration,
+ ),
+ budget_duration=resolved.budget_config.budget_duration,
+ now=self._clock(),
+ )
+ return local_spend if shared is None else max(local_spend, shared.spend)
+
async def _cached_spend(self, spend_key: str) -> float | None:
redis_cache: Final = self.dual_cache.redis_cache
if redis_cache is None:
@@ -669,7 +973,9 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
),
)
- resolved_budgets: Final = _resolve_entity_model_budgets(model=model, entity_budgets=entity_budgets)
+ resolved_budgets: Final = _resolve_entity_model_budgets(
+ model=model, entity_budgets=entity_budgets, model_access_groups=self._model_access_groups
+ )
if not resolved_budgets:
verbose_proxy_logger.debug(
"Not running _PROXY_VirtualKeyModelMaxBudgetLimiter.async_log_success_event: "
@@ -723,17 +1029,78 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
ttl_seconds=duration_in_seconds(budget_duration),
):
return
+ start_time_key: Final = model_budget_start_time_cache_key(
+ entity_type=entity_type,
+ entity_id=entity_id,
+ budget_model=resolved.budget_model,
+ budget_duration=budget_duration,
+ )
+ shares_window: Final = self._shares_windows_in_database(entity_type)
+ if shares_window:
+ await self._join_shared_window(
+ spend_key=spend_key, start_time_key=start_time_key, budget_duration=budget_duration
+ )
await self._increment_spend_for_key(
budget_config=resolved.budget_config,
spend_key=spend_key,
- start_time_key=model_budget_start_time_cache_key(
- entity_type=entity_type,
- entity_id=entity_id,
- budget_model=resolved.budget_model,
- budget_duration=budget_duration,
- ),
+ start_time_key=start_time_key,
response_cost=response_cost,
)
+ if shares_window:
+ await self._share_spend(
+ spend_key=spend_key,
+ start_time_key=start_time_key,
+ budget_duration=budget_duration,
+ response_cost=response_cost,
+ )
+
+ def _shares_windows_in_database(self, entity_type: Litellm_EntityType) -> bool:
+ """End-user ids are whatever the client sends, so their windows stay on each replica rather than adding a
+ database row per id."""
+ return self.dual_cache.redis_cache is None and entity_type not in _PER_REPLICA_SCOPES
+
+ async def _join_shared_window(self, spend_key: str, start_time_key: str, budget_duration: str) -> None:
+ """Without Redis, start this replica's window where the database says every replica's window starts.
+
+ Every replica then sends the same window start with its spend, so no flush reads as a newer window that
+ replaces the spend the others already added.
+ """
+ now: Final = self._clock()
+ local: Final = self.dual_cache.in_memory_cache
+ if _open_local_window_start(
+ local=local, start_time_key=start_time_key, budget_duration=budget_duration, now=now
+ ):
+ return
+ started_at: Final = await self._shared_windows.open(
+ spend_key=spend_key, budget_duration=budget_duration, now=now
+ )
+ if _open_local_window_start(
+ local=local, start_time_key=start_time_key, budget_duration=budget_duration, now=now
+ ):
+ return
+ _start_local_window(
+ local=local,
+ spend_key=spend_key,
+ start_time_key=start_time_key,
+ started_at=started_at,
+ budget_duration=budget_duration,
+ now=now,
+ )
+
+ async def _share_spend(
+ self, spend_key: str, start_time_key: str, budget_duration: str, response_cost: float
+ ) -> None:
+ now: Final = self._clock()
+ await self._shared_windows.add(
+ spend_key=spend_key,
+ budget_duration=budget_duration,
+ started_at=await self._get_or_set_budget_start_time(
+ start_time_key=start_time_key,
+ current_time=now,
+ ttl_seconds=budget_window_seconds(now, budget_duration),
+ ),
+ spend=response_cost,
+ )
async def _claim_batch_charge(self, spend_key: str, batch_id: str, ttl_seconds: int) -> bool:
marker_key: Final = batch_charged_once_marker_key(spend_key=spend_key, batch_id=batch_id)
@@ -743,4 +1110,17 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
return polls == 1
+def _open_local_window_start(local: LocalTTLCache, start_time_key: str, budget_duration: str, now: float) -> bool:
+ started_at: Final = _as_float(local.get_cache(key=start_time_key))
+ return started_at is not None and now - started_at <= budget_window_seconds(started_at, budget_duration)
+
+
+def _start_local_window(
+ local: LocalTTLCache, spend_key: str, start_time_key: str, started_at: float, budget_duration: str, now: float
+) -> None:
+ ttl: Final = started_at + budget_window_seconds(started_at, budget_duration) - now
+ local.set_cache(key=spend_key, value=0.0, ttl=ttl)
+ local.set_cache(key=start_time_key, value=started_at, ttl=ttl)
+
+
PROXY_VirtualKeyModelMaxBudgetLimiter = _PROXY_VirtualKeyModelMaxBudgetLimiter
diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py
index 1fa18e9b2b8..8cfdf81ea3a 100644
--- a/litellm/proxy/management_endpoints/team_endpoints.py
+++ b/litellm/proxy/management_endpoints/team_endpoints.py
@@ -139,7 +139,10 @@ from litellm.proxy.common_utils.user_api_key_cache import (
from litellm.proxy.db.db_span import db_span
from litellm.proxy.hooks.key_management_event_hooks import KeyManagementEventHooks
from litellm.proxy.hooks.model_max_budget_limiter import (
+ RoutedModels,
build_model_max_budget_usage,
+ entries_outranked_by,
+ proxy_routed_models,
resolve_model_budget,
)
from litellm.proxy.management.teams.authz import TEAM_OR_ORG_ADMIN, TeamRole, is_team_admin, team_access_denied
@@ -1467,6 +1470,7 @@ def _check_team_model_budget_update_authority(
data: UpdateTeamRequest,
user_api_key_dict: UserAPIKeyAuth,
existing_model_max_budget: Mapping[str, object] | None,
+ routed_models_source: Callable[[], RoutedModels] = proxy_routed_models,
) -> None:
"""Like `_check_team_budget_update_authority`: only a proxy admin may raise, re-window or drop a per-model cap."""
if user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN:
@@ -1494,11 +1498,7 @@ def _check_team_model_budget_update_authority(
cap = governing.budget_config
if cap.max_budget is None:
continue
- if (
- proposed.max_budget is None
- or proposed.max_budget > cap.max_budget
- or proposed.budget_duration != cap.budget_duration
- ):
+ if _looser_model_cap(proposed=proposed, cap=cap):
raise HTTPException(
status_code=403,
detail={
@@ -1510,6 +1510,74 @@ def _check_team_model_budget_update_authority(
)
},
)
+ routed_models: Final = routed_models_source()
+ for model_name, proposed in requested.items():
+ _check_outranked_model_caps(
+ entry=model_name,
+ proposed=proposed,
+ existing_model_max_budget=existing_model_max_budget,
+ routed_models=routed_models,
+ )
+
+
+def _check_outranked_model_caps(
+ entry: str,
+ proposed: BudgetConfig,
+ existing_model_max_budget: Mapping[str, object],
+ routed_models: RoutedModels,
+) -> None:
+ if _keeps_or_tightens(proposed=proposed, existing=_existing_model_cap(existing_model_max_budget.get(entry))):
+ return
+ outranked: Final = next(
+ _looser_than_outranked_caps(entry, proposed, existing_model_max_budget, routed_models), None
+ )
+ if outranked is None:
+ return
+ outranked_name, cap = outranked
+ raise HTTPException(
+ status_code=403,
+ detail={
+ "error": (
+ f"Only a proxy admin can set a team's model_max_budget entry {entry!r} looser than "
+ f"{outranked_name!r}, which it can take precedence over. Current max_budget={cap.max_budget} per "
+ f"{cap.budget_duration} (entry {outranked_name!r}), requested={proposed.max_budget} per "
+ f"{proposed.budget_duration}."
+ )
+ },
+ )
+
+
+def _looser_than_outranked_caps(
+ entry: str,
+ proposed: BudgetConfig,
+ existing_model_max_budget: Mapping[str, object],
+ routed_models: RoutedModels,
+) -> Iterator[tuple[str, BudgetConfig]]:
+ outranked_caps: Final = (
+ (outranked, _existing_model_cap(existing_model_max_budget[outranked]))
+ for outranked in entries_outranked_by(
+ entry=entry, model_max_budget=existing_model_max_budget, routed_models=routed_models
+ )
+ )
+ yield from (
+ (outranked, cap)
+ for outranked, cap in outranked_caps
+ if cap is not None and cap.max_budget is not None and _looser_model_cap(proposed=proposed, cap=cap)
+ )
+
+
+def _keeps_or_tightens(proposed: BudgetConfig, existing: BudgetConfig | None) -> bool:
+ return existing is not None and (
+ existing.max_budget is None or not _looser_model_cap(proposed=proposed, cap=existing)
+ )
+
+
+def _looser_model_cap(proposed: BudgetConfig, cap: BudgetConfig) -> bool:
+ return (
+ proposed.max_budget is None
+ or (cap.max_budget is not None and proposed.max_budget > cap.max_budget)
+ or proposed.budget_duration != cap.budget_duration
+ )
def _should_auto_add_team_creator(
diff --git a/tests/unit/proxy/db/test_budget_window_spend_writer.py b/tests/unit/proxy/db/test_budget_window_spend_writer.py
index 6fc438feee8..27f1472c183 100644
--- a/tests/unit/proxy/db/test_budget_window_spend_writer.py
+++ b/tests/unit/proxy/db/test_budget_window_spend_writer.py
@@ -9,6 +9,7 @@ import pytest
from litellm.proxy.db.budget_window_spend_writer import (
WindowSeedTotals,
commit_window_spend_updates,
+ open_window_spend_row,
roll_window_spend_row,
spend_logs_seed_totals,
)
@@ -405,6 +406,29 @@ async def test_roll_window_spend_row_is_conditional_on_the_stored_window_being_o
assert params[:4] == ("team", "t1", "30d", datetime(2026, 8, 31))
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+ ("replaces_window_start", "replaced_param"), [(None, None), (WINDOW_A, datetime(2026, 8, 1))]
+)
+async def test_open_window_spend_row_moves_only_the_window_the_caller_saw_end(replaces_window_start, replaced_param):
+ db = _FakeDB()
+
+ await open_window_spend_row(
+ prisma_client=_FakePrismaClient(db),
+ entity_type="model_max_budget",
+ entity_id="k1",
+ window_duration="1d",
+ window_start=WINDOW_B,
+ replaces_window_start=replaces_window_start,
+ )
+
+ ((query, params),) = db.execute_raw_calls
+ normalized = " ".join(query.split())
+ assert "ON CONFLICT (entity_type, entity_id, window_duration) DO UPDATE" in normalized
+ assert normalized.endswith("WHERE \"LiteLLM_BudgetWindowSpend\".window_start <= ($5::timestamptz AT TIME ZONE 'UTC')")
+ assert params == ("model_max_budget", "k1", "1d", datetime(2026, 8, 31), replaced_param)
+
+
@pytest.mark.asyncio
async def test_seed_receives_the_batch_earliest_start_as_its_cutoff():
db = _FakeDB(existing_rows=[])
diff --git a/tests/unit/proxy/hooks/test_model_budget_shared_windows.py b/tests/unit/proxy/hooks/test_model_budget_shared_windows.py
new file mode 100644
index 00000000000..c294d667315
--- /dev/null
+++ b/tests/unit/proxy/hooks/test_model_budget_shared_windows.py
@@ -0,0 +1,193 @@
+from datetime import datetime, timedelta, timezone
+from types import SimpleNamespace
+from typing import Final
+
+import pytest
+
+from litellm.proxy.db.db_transaction_queue.window_spend_update_queue import WindowSpendTransaction
+from litellm.proxy.hooks.model_budget_shared_windows import DatabaseSharedWindows, SharedWindow
+
+SPEND_KEY: Final = "virtual_key_spend:key-hash:claude-opus*:1d"
+WINDOW_START: Final = datetime(2026, 10, 20, 9, 30, tzinfo=timezone.utc)
+
+
+class _Table:
+ def __init__(self, row: object | None = None, error: Exception | None = None) -> None:
+ self.row = row
+ self.error = error
+ self.reads = 0
+
+ async def find_unique(self, where: object) -> object | None:
+ self.reads = self.reads + 1
+ if self.error is not None:
+ raise self.error
+ return self.row
+
+
+class _Database:
+ def __init__(self, table: _Table) -> None:
+ self.litellm_budgetwindowspend = table
+ self.opened_with: tuple[object, ...] = ()
+
+ async def execute_raw(self, query: str, *params: object) -> int:
+ self.opened_with = params
+ self.litellm_budgetwindowspend.row = SimpleNamespace(window_start=params[3], spend=0.0)
+ return 1
+
+
+class _OutracedDatabase:
+ def __init__(self, table: _Table, winner: object) -> None:
+ self.litellm_budgetwindowspend = table
+ self._winner = winner
+
+ async def execute_raw(self, query: str, *params: object) -> int:
+ self.litellm_budgetwindowspend.row = self._winner
+ return 0
+
+
+class _Sink:
+ def __init__(self) -> None:
+ self.transactions: tuple[WindowSpendTransaction, ...] = ()
+
+ async def __call__(self, transaction: WindowSpendTransaction) -> None:
+ self.transactions = (*self.transactions, transaction)
+
+
+class _LaggingReplica:
+ def __init__(self, writer: _Database) -> None:
+ self.litellm_budgetwindowspend = _Table()
+ self._writer = writer
+
+ async def execute_raw(self, query: str, *params: object) -> int:
+ return await self._writer.execute_raw(query, *params)
+
+
+def _database_windows(table: _Table | None, sink: _Sink, lagging_replica: bool = False) -> DatabaseSharedWindows:
+ writer: Final = None if table is None else _Database(table)
+ prisma: Final = (
+ None
+ if writer is None
+ else SimpleNamespace(db=_LaggingReplica(writer) if lagging_replica else writer, writer_db=writer)
+ )
+ return DatabaseSharedWindows(
+ prisma_client_source=lambda: prisma, # pyright: ignore[reportArgumentType] # duck-typed prisma double
+ sink=sink,
+ )
+
+
+def test_a_window_is_open_until_its_period_has_passed():
+ window: Final = SharedWindow(started_at=WINDOW_START.timestamp(), spend=1.0)
+ day: Final = 24 * 60 * 60
+
+ assert window.is_open(budget_duration="1d", now=WINDOW_START.timestamp() + day - 1)
+ assert not window.is_open(budget_duration="1d", now=WINDOW_START.timestamp() + day)
+
+
+@pytest.mark.asyncio
+async def test_read_returns_the_stored_window_and_rereads_only_after_the_cache_expires():
+ table: Final = _Table(row=SimpleNamespace(window_start=WINDOW_START.replace(tzinfo=None), spend=2.5))
+ windows: Final = _database_windows(table, _Sink())
+
+ first: Final = await windows.read(spend_key=SPEND_KEY, budget_duration="1d")
+ second: Final = await windows.read(spend_key=SPEND_KEY, budget_duration="1d")
+
+ assert first == SharedWindow(started_at=WINDOW_START.timestamp(), spend=2.5)
+ assert second == first
+ assert table.reads == 1
+
+
+@pytest.mark.asyncio
+async def test_an_unreadable_table_reads_as_no_shared_window():
+ table: Final = _Table(error=RuntimeError("database is down"))
+ windows: Final = _database_windows(table, _Sink())
+
+ assert await windows.read(spend_key=SPEND_KEY, budget_duration="1d") is None
+
+
+@pytest.mark.asyncio
+async def test_add_queues_the_charge_for_the_window_it_belongs_to():
+ sink: Final = _Sink()
+ windows: Final = _database_windows(_Table(), sink)
+
+ await windows.add(spend_key=SPEND_KEY, budget_duration="1d", started_at=WINDOW_START.timestamp(), spend=0.25)
+
+ assert len(sink.transactions) == 1
+ queued: Final = sink.transactions[0]
+ assert (queued["entity_id"], queued["window_duration"], queued["spend"]) == (SPEND_KEY, "1d", 0.25)
+ assert datetime.fromisoformat(queued["window_start"]).replace(tzinfo=timezone.utc) == WINDOW_START
+
+
+@pytest.mark.asyncio
+async def test_nothing_is_queued_without_a_database():
+ sink: Final = _Sink()
+ windows: Final = _database_windows(None, sink)
+
+ await windows.add(spend_key=SPEND_KEY, budget_duration="1d", started_at=WINDOW_START.timestamp(), spend=0.25)
+
+ assert sink.transactions == ()
+ assert await windows.read(spend_key=SPEND_KEY, budget_duration="1d") is None
+
+
+@pytest.mark.asyncio
+async def test_open_joins_the_window_another_replica_already_started():
+ table: Final = _Table(row=SimpleNamespace(window_start=WINDOW_START.replace(tzinfo=None), spend=2.5))
+ windows: Final = _database_windows(table, _Sink())
+
+ started_at: Final = await windows.open(
+ spend_key=SPEND_KEY, budget_duration="1d", now=WINDOW_START.timestamp() + 3600
+ )
+
+ assert started_at == WINDOW_START.timestamp()
+ assert table.row.spend == 2.5
+
+
+@pytest.mark.asyncio
+async def test_open_reads_the_window_from_the_writer_while_the_read_replica_lags():
+ table: Final = _Table(row=SimpleNamespace(window_start=WINDOW_START.replace(tzinfo=None), spend=2.5))
+ windows: Final = _database_windows(table, _Sink(), lagging_replica=True)
+
+ started_at: Final = await windows.open(
+ spend_key=SPEND_KEY, budget_duration="1d", now=WINDOW_START.timestamp() + 3600
+ )
+
+ assert started_at == WINDOW_START.timestamp()
+
+
+@pytest.mark.asyncio
+async def test_open_joins_the_window_a_racing_replica_opened_first():
+ winner: Final = SimpleNamespace(window_start=WINDOW_START.replace(tzinfo=None), spend=2.5)
+ database: Final = _OutracedDatabase(_Table(), winner)
+ windows: Final = DatabaseSharedWindows(
+ prisma_client_source=lambda: SimpleNamespace(db=database, writer_db=database), # pyright: ignore[reportArgumentType] # duck-typed prisma double
+ sink=_Sink(),
+ )
+
+ started_at: Final = await windows.open(spend_key=SPEND_KEY, budget_duration="1d", now=WINDOW_START.timestamp() + 60)
+
+ assert started_at == WINDOW_START.timestamp()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("stored_start", [None, WINDOW_START - timedelta(days=2)])
+async def test_open_starts_a_window_at_now_when_none_is_open(stored_start: datetime | None):
+ table: Final = _Table(
+ row=None if stored_start is None else SimpleNamespace(window_start=stored_start.replace(tzinfo=None), spend=9.0)
+ )
+ windows: Final = _database_windows(table, _Sink())
+
+ started_at: Final = await windows.open(spend_key=SPEND_KEY, budget_duration="1d", now=WINDOW_START.timestamp())
+
+ assert started_at == WINDOW_START.timestamp()
+ assert await windows.read(spend_key=SPEND_KEY, budget_duration="1d") == SharedWindow(
+ started_at=WINDOW_START.timestamp(), spend=0.0
+ )
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("table", [None, _Table(error=RuntimeError("database is down"))])
+async def test_open_falls_back_to_this_replicas_own_window_without_a_database(table: _Table | None):
+ windows: Final = _database_windows(table, _Sink())
+
+ assert await windows.open(spend_key=SPEND_KEY, budget_duration="1d", now=WINDOW_START.timestamp()) == (
+ WINDOW_START.timestamp()
+ )
diff --git a/tests/unit/proxy/hooks/test_model_max_budget_limiter.py b/tests/unit/proxy/hooks/test_model_max_budget_limiter.py
index 2678f4205d5..e97be8abbb0 100644
--- a/tests/unit/proxy/hooks/test_model_max_budget_limiter.py
+++ b/tests/unit/proxy/hooks/test_model_max_budget_limiter.py
@@ -1,6 +1,9 @@
import asyncio
+import functools
+import re
import time
from collections.abc import Callable, Mapping
+from datetime import datetime, timezone
from types import MappingProxyType
from typing import Final
@@ -8,12 +11,19 @@ import pytest
import litellm
from litellm.caching.caching import DualCache
-from litellm.proxy._types import Litellm_EntityType
+from litellm.proxy._types import Litellm_EntityType, UserAPIKeyAuth
+from litellm.proxy.hooks.model_budget_shared_windows import SharedWindow, SharedWindows
from litellm.proxy.hooks.model_max_budget_limiter import (
PROXY_VirtualKeyModelMaxBudgetLimiter,
+ build_model_max_budget_usage,
+ model_access_groups_of,
model_budget_spend_cache_key,
+ resolve_model_budget,
+ routed_models_of,
team_member_budget_entity_id,
+ team_model_budget_applies,
)
+from litellm.router import Router
from litellm.types.caching import RedisPipelineIncrementOperation
from litellm.types.utils import LiteLLMBatch, Usage
@@ -74,10 +84,14 @@ def _event(
}
+async def _log_success(
+ limiter: PROXY_VirtualKeyModelMaxBudgetLimiter, event: dict[str, object], response_obj: object = None
+) -> None:
+ await limiter.async_log_success_event(event, response_obj=response_obj, start_time=None, end_time=None)
+
+
async def _poll(limiter: PROXY_VirtualKeyModelMaxBudgetLimiter, batch: LiteLLMBatch, response_cost: float) -> None:
- await limiter.async_log_success_event(
- _event("aretrieve_batch", response_cost), response_obj=batch, start_time=None, end_time=None
- )
+ await _log_success(limiter, _event("aretrieve_batch", response_cost), response_obj=batch)
async def _chat(
@@ -86,7 +100,8 @@ async def _chat(
user_id: str = USER_ID,
team_id: str = TEAM_ID,
) -> None:
- await limiter.async_log_success_event(
+ await _log_success(
+ limiter,
_event(
"acompletion",
CHAT_COST,
@@ -94,9 +109,6 @@ async def _chat(
user_id=user_id,
team_id=team_id,
),
- response_obj=None,
- start_time=None,
- end_time=None,
)
@@ -311,3 +323,399 @@ async def test_a_batch_polled_within_every_budget_window_is_never_charged_again(
await _poll(limiter, finished, BATCH_COST)
assert _local_spend(limiter, KEY_SPEND_KEY) == pytest.approx(BATCH_COST)
+
+
+OPUS_GROUP: Final = "claude-opus"
+OPUS_GROUP_MEMBERS: Final = MappingProxyType(
+ {"claude-opus-4-8": frozenset({OPUS_GROUP}), "claude-opus-5-5": frozenset({OPUS_GROUP})}
+)
+FAMILY_CAP: Final = {"budget_limit": CHAT_COST, "time_period": "1d"} # mutable-ok: BudgetConfig validates dicts only
+ROOMY_CAP: Final = {
+ "budget_limit": CHAT_COST * 100,
+ "time_period": "1d",
+} # mutable-ok: BudgetConfig validates dicts only
+WINDOW_EPOCH: Final = datetime(2026, 10, 20, 9, 30, tzinfo=timezone.utc).timestamp()
+
+
+def _model_access_groups(groups_by_model: Mapping[str, frozenset[str]]) -> Callable[[str], frozenset[str]]:
+ return lambda model: groups_by_model.get(model, frozenset())
+
+
+class _SharedWindowsDouble:
+ def __init__(self) -> None:
+ self.rows: Mapping[str, SharedWindow] = MappingProxyType({})
+ self.queued: tuple[tuple[str, float, float], ...] = ()
+
+ async def read(self, spend_key: str, budget_duration: str) -> SharedWindow | None:
+ return self.rows.get(spend_key)
+
+ async def open(self, spend_key: str, budget_duration: str, now: float) -> float:
+ await asyncio.sleep(0)
+ stored: Final = self.rows.get(spend_key)
+ if stored is not None and stored.is_open(budget_duration=budget_duration, now=now):
+ return stored.started_at
+ self.rows = MappingProxyType({**self.rows, spend_key: SharedWindow(started_at=now, spend=0.0)})
+ return now
+
+ async def add(self, spend_key: str, budget_duration: str, started_at: float, spend: float) -> None:
+ self.queued = (*self.queued, (spend_key, started_at, spend))
+
+ def flush(self) -> None:
+ self.rows = functools.reduce(_upsert_window_spend, self.queued, self.rows)
+ self.queued = ()
+
+
+def _upsert_window_spend(
+ rows: Mapping[str, SharedWindow], queued: tuple[str, float, float]
+) -> Mapping[str, SharedWindow]:
+ spend_key, started_at, spend = queued
+ stored: Final = rows.get(spend_key)
+ row: Final = (
+ SharedWindow(started_at=stored.started_at, spend=stored.spend + spend)
+ if stored is not None and stored.started_at >= started_at
+ else SharedWindow(started_at=started_at, spend=spend)
+ )
+ return MappingProxyType({**rows, spend_key: row})
+
+
+def _family_limiter(
+ groups_by_model: Mapping[str, frozenset[str]] = OPUS_GROUP_MEMBERS,
+ dual_cache: DualCache | None = None,
+ shared_windows: SharedWindows | None = None,
+ clock: Callable[[], float] = time.time,
+) -> PROXY_VirtualKeyModelMaxBudgetLimiter:
+ return PROXY_VirtualKeyModelMaxBudgetLimiter(
+ dual_cache=dual_cache or DualCache(),
+ clock=clock,
+ model_access_groups=_model_access_groups(groups_by_model),
+ shared_windows=shared_windows or _SharedWindowsDouble(),
+ )
+
+
+def _key(model_max_budget: Mapping[str, object]) -> UserAPIKeyAuth:
+ return UserAPIKeyAuth(token=KEY_HASH, key_alias="opus-family", model_max_budget=dict(model_max_budget))
+
+
+async def _spend_on(
+ limiter: PROXY_VirtualKeyModelMaxBudgetLimiter, model: str, model_max_budget: Mapping[str, object]
+) -> None:
+ await _log_success(
+ limiter,
+ {
+ "call_type": "acompletion",
+ "standard_logging_object": {
+ "call_type": "acompletion",
+ "response_cost": CHAT_COST,
+ "model": f"anthropic/{model}",
+ "model_group": model,
+ "metadata": {"user_api_key_hash": KEY_HASH},
+ },
+ "litellm_params": {"metadata": {"user_api_key_model_max_budget": dict(model_max_budget)}},
+ },
+ )
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("family_entry", ["claude-opus*", OPUS_GROUP])
+async def test_every_opus_version_draws_from_one_family_entry(family_entry: str):
+ budget: Final = {family_entry: FAMILY_CAP}
+ limiter: Final = _family_limiter()
+ assert await limiter.is_key_within_model_budget(user_api_key_dict=_key(budget), model="claude-opus-5-5")
+
+ await _spend_on(limiter, model="claude-opus-4-8", model_max_budget=budget)
+
+ with pytest.raises(
+ litellm.BudgetExceededError, match=rf"model=claude-opus-5-5, budget_model={re.escape(family_entry)}$"
+ ):
+ await limiter.is_key_within_model_budget(user_api_key_dict=_key(budget), model="claude-opus-5-5")
+ assert await limiter.is_key_within_model_budget(user_api_key_dict=_key(budget), model="claude-sonnet-5")
+
+
+@pytest.mark.asyncio
+async def test_a_model_added_to_the_access_group_later_is_covered_with_no_key_change():
+ budget: Final = {OPUS_GROUP: FAMILY_CAP}
+ shared_cache: Final = DualCache()
+ before_group_change: Final = _family_limiter(dual_cache=shared_cache)
+ after_group_change: Final = _family_limiter(
+ groups_by_model={**OPUS_GROUP_MEMBERS, "claude-opus-6": frozenset({OPUS_GROUP})}, dual_cache=shared_cache
+ )
+
+ await _spend_on(before_group_change, model="claude-opus-4-8", model_max_budget=budget)
+
+ assert await before_group_change.is_key_within_model_budget(user_api_key_dict=_key(budget), model="claude-opus-6")
+ with pytest.raises(litellm.BudgetExceededError):
+ await after_group_change.is_key_within_model_budget(user_api_key_dict=_key(budget), model="claude-opus-6")
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+ ("budget", "checked_model", "blocked"),
+ [
+ ({"claude-opus-4-8": ROOMY_CAP, "claude-opus*": FAMILY_CAP}, "claude-opus-4-8", False),
+ ({"claude-opus-4-8": ROOMY_CAP, "claude-opus*": FAMILY_CAP}, "claude-opus-5-5", False),
+ ({OPUS_GROUP: FAMILY_CAP, "claude-opus*": ROOMY_CAP}, "claude-opus-5-5", True),
+ ({"claude-opus-4*": FAMILY_CAP, "claude-opus*": ROOMY_CAP}, "claude-opus-4-7", True),
+ ({"claude-opus-4*": FAMILY_CAP, "claude-opus*": ROOMY_CAP}, "claude-opus-5-5", False),
+ ({"claude-opus-4*": {"budget_limit": CHAT_COST}, "claude-opus*": FAMILY_CAP}, "claude-opus-5-5", True),
+ ({"claude-opus-4*": FAMILY_CAP, "c" + "*" * 15: ROOMY_CAP}, "claude-opus-4-7", True),
+ ],
+)
+async def test_the_most_specific_entry_takes_the_spend_and_gates_the_request(
+ budget: Mapping[str, object], checked_model: str, blocked: bool
+):
+ limiter: Final = _family_limiter()
+
+ await _spend_on(limiter, model="claude-opus-4-8", model_max_budget=budget)
+
+ if blocked:
+ with pytest.raises(litellm.BudgetExceededError):
+ await limiter.is_key_within_model_budget(user_api_key_dict=_key(budget), model=checked_model)
+ else:
+ assert await limiter.is_key_within_model_budget(user_api_key_dict=_key(budget), model=checked_model)
+
+
+@pytest.mark.asyncio
+async def test_an_exact_entry_refusal_reads_as_before_for_a_provider_prefixed_model():
+ budget: Final = {"claude-opus-4-8": FAMILY_CAP}
+ limiter: Final = _family_limiter()
+
+ await _spend_on(limiter, model="claude-opus-4-8", model_max_budget=budget)
+
+ with pytest.raises(litellm.BudgetExceededError, match=r"model=anthropic/claude-opus-4-8$"):
+ await limiter.is_key_within_model_budget(user_api_key_dict=_key(budget), model="anthropic/claude-opus-4-8")
+
+
+@pytest.mark.parametrize(
+ ("key_entry", "team_cap_applies"),
+ [("claude-opus-4-8", False), ("anthropic/claude-opus-4-8", True), ("claude-opus*", True), (OPUS_GROUP, True)],
+)
+def test_only_a_key_entry_named_after_the_model_replaces_the_team_cap(key_entry: str, team_cap_applies: bool):
+ assert team_model_budget_applies(model="claude-opus-4-8", key_model_max_budget={key_entry: ROOMY_CAP}) is (
+ team_cap_applies
+ )
+
+
+@pytest.mark.asyncio
+async def test_end_user_windows_stay_on_the_replica_without_redis():
+ budget: Final = {"claude-opus*": FAMILY_CAP}
+ table: Final = _SharedWindowsDouble()
+ limiter: Final = _family_limiter(shared_windows=table)
+
+ await _log_success(
+ limiter,
+ {
+ "call_type": "acompletion",
+ "standard_logging_object": {
+ "call_type": "acompletion",
+ "response_cost": CHAT_COST,
+ "model": "anthropic/claude-opus-4-8",
+ "model_group": "claude-opus-4-8",
+ "end_user": "end-user-1",
+ "metadata": {},
+ },
+ "litellm_params": {"metadata": {"user_api_key_end_user_model_max_budget": budget}},
+ },
+ )
+ table.flush()
+
+ assert dict(table.rows) == {}
+ with pytest.raises(litellm.BudgetExceededError, match="model=claude-opus-5-5"):
+ await limiter.is_end_user_within_model_budget(
+ end_user_id="end-user-1", end_user_model_max_budget=budget, model="claude-opus-5-5"
+ )
+
+
+def test_an_access_group_named_after_a_served_model_never_widens_that_models_entry():
+ router: Final = Router(
+ model_list=[
+ {
+ "model_name": "gpt-4o",
+ "litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-test"},
+ "model_info": {"access_groups": ["gpt-4o", "openai-models"]},
+ },
+ {
+ "model_name": "gpt-4o-mini",
+ "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-test"},
+ "model_info": {"access_groups": ["gpt-4o", "openai-models"]},
+ },
+ ]
+ )
+
+ def governing_entry(model_max_budget: Mapping[str, object]) -> str | None:
+ resolved: Final = resolve_model_budget(
+ model="gpt-4o-mini",
+ model_max_budget=model_max_budget,
+ model_access_groups=lambda model: model_access_groups_of(router=router, model=model),
+ )
+ return None if resolved is None else resolved.budget_model
+
+ assert governing_entry({"gpt-4o": FAMILY_CAP}) is None
+ assert governing_entry({"openai-models": FAMILY_CAP}) == "openai-models"
+ assert routed_models_of(router).members("gpt-4o") == frozenset()
+ assert routed_models_of(router).members("openai-models") == frozenset({"gpt-4o", "gpt-4o-mini"})
+
+
+@pytest.mark.asyncio
+async def test_key_info_usage_reports_the_family_total():
+ budget: Final = {"claude-opus*": ROOMY_CAP}
+ limiter: Final = _family_limiter()
+
+ await _spend_on(limiter, model="claude-opus-4-8", model_max_budget=budget)
+ await _spend_on(limiter, model="claude-opus-5-5", model_max_budget=budget)
+
+ usage: Final = await build_model_max_budget_usage(
+ entity_type=Litellm_EntityType.KEY,
+ entity_id=KEY_HASH,
+ model_max_budget=budget,
+ cache=limiter.dual_cache,
+ )
+ assert usage["claude-opus*"]["current_spend"] == round(CHAT_COST * 2, 4)
+
+
+@pytest.mark.parametrize(
+ ("pattern", "model", "governs"),
+ [
+ ("claude-opus*", "claude-opus-4-8", True),
+ ("claude-opus*", "anthropic/claude-opus-4-8", True),
+ ("anthropic/claude-opus*", "anthropic/claude-opus-4-8", True),
+ ("anthropic/claude-opus*", "claude-opus-4-8", False),
+ ("claude-opus*", "my-claude-opus-4-8", False),
+ ("claude-opus*", "claude-sonnet-5", False),
+ ("gpt-4.1*", "gpt-401-mini", False),
+ ("*", "claude-sonnet-5", True),
+ ],
+)
+def test_glob_entries_match_the_whole_model_name_with_only_star_as_a_wildcard(pattern: str, model: str, governs: bool):
+ resolved: Final = resolve_model_budget(
+ model=model, model_max_budget={pattern: FAMILY_CAP}, model_access_groups=_model_access_groups({})
+ )
+
+ assert (resolved is not None and resolved.budget_model == pattern) is governs
+
+
+@pytest.mark.asyncio
+async def test_replicas_without_redis_enforce_one_family_cap_together():
+ budget: Final = {"claude-opus*": FAMILY_CAP}
+ table: Final = _SharedWindowsDouble()
+ replica_a: Final = _family_limiter(shared_windows=table)
+ replica_b: Final = _family_limiter(shared_windows=table)
+
+ await _spend_on(replica_a, model="claude-opus-4-8", model_max_budget=budget)
+ table.flush()
+
+ with pytest.raises(litellm.BudgetExceededError, match="model=claude-opus-5-5"):
+ await replica_b.is_key_within_model_budget(user_api_key_dict=_key(budget), model="claude-opus-5-5")
+
+
+@pytest.mark.asyncio
+async def test_replicas_sharing_redis_enforce_the_family_cap_without_the_database():
+ budget: Final = {"claude-opus*": FAMILY_CAP}
+ redis: Final = _SharedRedisDouble()
+ table: Final = _SharedWindowsDouble()
+ replica_a: Final = _family_limiter(
+ dual_cache=DualCache(redis_cache=redis), # pyright: ignore[reportArgumentType] # duck-typed Redis double
+ shared_windows=table,
+ )
+ replica_b: Final = _family_limiter(
+ dual_cache=DualCache(redis_cache=redis), # pyright: ignore[reportArgumentType] # duck-typed Redis double
+ shared_windows=table,
+ )
+
+ await _spend_on(replica_a, model="claude-opus-4-8", model_max_budget=budget)
+ await _drain_redis_pushes()
+ table.flush()
+
+ with pytest.raises(litellm.BudgetExceededError, match="model=claude-opus-5-5"):
+ await replica_b.is_key_within_model_budget(user_api_key_dict=_key(budget), model="claude-opus-5-5")
+ assert dict(table.rows) == {}
+
+
+@pytest.mark.asyncio
+async def test_a_replica_charges_the_open_shared_window_instead_of_restarting_it():
+ budget: Final = {"claude-opus*": {"budget_limit": CHAT_COST * 1.5, "time_period": "1d"}}
+ table: Final = _SharedWindowsDouble()
+ clock: Final = _Clock()
+ clock.advance(WINDOW_EPOCH)
+ replica_a: Final = _family_limiter(shared_windows=table, clock=clock.now)
+ replica_b: Final = _family_limiter(shared_windows=table, clock=clock.now)
+
+ await _spend_on(replica_a, model="claude-opus-4-8", model_max_budget=budget)
+ table.flush()
+ clock.advance(3600)
+ await _spend_on(replica_b, model="claude-opus-5-5", model_max_budget=budget)
+ table.flush()
+
+ for replica in (replica_a, replica_b):
+ with pytest.raises(litellm.BudgetExceededError):
+ await replica.is_key_within_model_budget(user_api_key_dict=_key(budget), model="claude-opus-5-5")
+
+
+@pytest.mark.asyncio
+async def test_key_info_on_one_replica_reports_spend_from_every_replica():
+ budget: Final = {"claude-opus*": ROOMY_CAP}
+ table: Final = _SharedWindowsDouble()
+ replica_a: Final = _family_limiter(shared_windows=table)
+ replica_b: Final = _family_limiter(shared_windows=table)
+
+ await _spend_on(replica_a, model="claude-opus-4-8", model_max_budget=budget)
+ await _spend_on(replica_b, model="claude-opus-5-5", model_max_budget=budget)
+ table.flush()
+
+ usage: Final = await build_model_max_budget_usage(
+ entity_type=Litellm_EntityType.KEY,
+ entity_id=KEY_HASH,
+ model_max_budget=budget,
+ cache=replica_a.dual_cache,
+ shared_windows=table,
+ )
+ assert usage["claude-opus*"]["current_spend"] == round(CHAT_COST * 2, 4)
+ assert usage["claude-opus*"]["budget_reset_at"] is not None
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("budget_model", ["claude-opus*", "claude-opus-4-8"])
+async def test_replicas_starting_a_window_moments_apart_keep_each_others_spend(budget_model: str):
+ budget: Final = {budget_model: ROOMY_CAP}
+ table: Final = _SharedWindowsDouble()
+ clock: Final = _Clock()
+ clock.advance(WINDOW_EPOCH)
+ replica_a: Final = _family_limiter(shared_windows=table, clock=clock.now)
+ replica_b: Final = _family_limiter(shared_windows=table, clock=clock.now)
+
+ await _spend_on(replica_a, model="claude-opus-4-8", model_max_budget=budget)
+ clock.advance(1)
+ await _spend_on(replica_b, model="claude-opus-4-8", model_max_budget=budget)
+ table.flush()
+
+ usage: Final = await build_model_max_budget_usage(
+ entity_type=Litellm_EntityType.KEY,
+ entity_id=KEY_HASH,
+ model_max_budget=budget,
+ cache=DualCache(),
+ now=datetime.fromtimestamp(clock.now(), tz=timezone.utc),
+ shared_windows=table,
+ )
+ assert usage[budget_model]["current_spend"] == round(CHAT_COST * 2, 4)
+
+
+@pytest.mark.asyncio
+async def test_concurrent_first_charges_on_one_replica_both_count():
+ budget: Final = {"claude-opus*": ROOMY_CAP}
+ clock: Final = _Clock()
+ clock.advance(WINDOW_EPOCH)
+ limiter: Final = _family_limiter(shared_windows=_SharedWindowsDouble(), clock=clock.now)
+
+ await asyncio.gather(
+ _spend_on(limiter, model="claude-opus-4-8", model_max_budget=budget),
+ _spend_on(limiter, model="claude-opus-5-5", model_max_budget=budget),
+ )
+
+ usage: Final = await build_model_max_budget_usage(
+ entity_type=Litellm_EntityType.KEY,
+ entity_id=KEY_HASH,
+ model_max_budget=budget,
+ cache=limiter.dual_cache,
+ now=datetime.fromtimestamp(clock.now(), tz=timezone.utc),
+ shared_windows=_SharedWindowsDouble(),
+ )
+ assert usage["claude-opus*"]["current_spend"] == round(CHAT_COST * 2, 4)
diff --git a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py
index 9634f1efce8..b13a7bfc33e 100644
--- a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py
+++ b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py
@@ -16580,6 +16580,7 @@ async def test_info_key_fn_includes_model_max_budget_usage(monkeypatch):
}
mock_prisma_client = AsyncMock()
+ mock_prisma_client.db.litellm_budgetwindowspend.find_unique = AsyncMock(return_value=None)
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client)
mock_user_api_key_cache = AsyncMock()
monkeypatch.setattr(
@@ -16703,6 +16704,7 @@ async def test_info_key_fn_v2_includes_model_max_budget_usage(monkeypatch):
model_max_budget = {"gpt-4o": {"budget_limit": 1.00, "time_period": "7d"}}
mock_prisma_client = AsyncMock()
+ mock_prisma_client.db.litellm_budgetwindowspend.find_unique = AsyncMock(return_value=None)
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client)
mock_user_api_key_cache = AsyncMock()
monkeypatch.setattr(
@@ -16767,6 +16769,7 @@ async def test_info_key_fn_budget_table_fallback(monkeypatch):
}
mock_prisma_client = AsyncMock()
+ mock_prisma_client.db.litellm_budgetwindowspend.find_unique = AsyncMock(return_value=None)
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client)
mock_user_api_key_cache = AsyncMock()
monkeypatch.setattr(
@@ -16840,6 +16843,7 @@ async def test_info_key_fn_v2_budget_table_fallback(monkeypatch):
}
mock_prisma_client = AsyncMock()
+ mock_prisma_client.db.litellm_budgetwindowspend.find_unique = AsyncMock(return_value=None)
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client)
mock_user_api_key_cache = AsyncMock()
monkeypatch.setattr(
@@ -17215,6 +17219,7 @@ async def test_info_key_fn_reads_the_configured_budget_model_key(monkeypatch):
}
mock_prisma_client = AsyncMock()
+ mock_prisma_client.db.litellm_budgetwindowspend.find_unique = AsyncMock(return_value=None)
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client)
mock_user_api_key_cache = AsyncMock()
monkeypatch.setattr(
diff --git a/tests/unit/proxy/management_endpoints/test_team_endpoints.py b/tests/unit/proxy/management_endpoints/test_team_endpoints.py
index f727448f3ac..6abc059384b 100644
--- a/tests/unit/proxy/management_endpoints/test_team_endpoints.py
+++ b/tests/unit/proxy/management_endpoints/test_team_endpoints.py
@@ -44,6 +44,7 @@ from litellm.proxy._types import (
UpdateTeamRequest,
UserAPIKeyAuth, # Import UserAPIKeyAuth
)
+from litellm.proxy.hooks.model_max_budget_limiter import RoutedModels
from litellm.proxy.management.teams.authz import TeamAccess
from litellm.proxy.management_endpoints.team_admin_field_permissions import (
PROXY_CONTROLLED_TEAM_METADATA_KEYS,
@@ -17593,6 +17594,130 @@ def test_team_admin_can_tighten_or_keep_team_model_caps(requested) -> None:
)
+_TEAM_OPUS_FAMILY_CAP: Final = MappingProxyType({"claude-opus*": {"max_budget": 10.0, "budget_duration": "1d"}})
+_TEAM_OPUS_GROUP_CAP: Final = MappingProxyType({"z-opus-group": {"max_budget": 10.0, "budget_duration": "1d"}})
+_TEAM_ROUTED_MODELS: Final = RoutedModels(
+ served_names=frozenset(
+ {"claude-opus-4-8", "claude-opus-5-5", "gemini-3-pro", "gpt-4o", "my-team/gpt-4o", "team-route/*"}
+ ),
+ group_members=MappingProxyType(
+ {
+ "opus-group": frozenset({"claude-opus-4-8"}),
+ "z-opus-group": frozenset({"claude-opus-4-8"}),
+ "a-opus-group": frozenset({"claude-opus-4-8", "claude-opus-5-5"}),
+ "zz-opus-group": frozenset({"claude-opus-4-8"}),
+ "a-gemini-group": frozenset({"gemini-3-pro"}),
+ "z-anthropic-group": frozenset({"anthropic/*"}),
+ }
+ ),
+)
+
+
+def _check_team_admin_model_caps(existing: Mapping[str, object], requested: Mapping[str, object]) -> None:
+ from litellm.proxy.management_endpoints.team_endpoints import _check_team_model_budget_update_authority
+
+ _check_team_model_budget_update_authority(
+ data=UpdateTeamRequest(team_id="t1", model_max_budget=dict(requested)),
+ user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, user_id="team-admin"),
+ existing_model_max_budget=existing,
+ routed_models_source=lambda: _TEAM_ROUTED_MODELS,
+ )
+
+
+@pytest.mark.parametrize(
+ ("existing", "added"),
+ [
+ (_TEAM_OPUS_FAMILY_CAP, {"opus-group": {"max_budget": 1000.0, "budget_duration": "1d"}}),
+ (_TEAM_OPUS_FAMILY_CAP, {"opus-group": {"max_budget": 10.0, "budget_duration": "7d"}}),
+ (_TEAM_OPUS_FAMILY_CAP, {"*laude-opus-4-8": {"max_budget": 1000.0, "budget_duration": "1d"}}),
+ (_TEAM_OPUS_GROUP_CAP, {"a-opus-group": {"max_budget": 1000.0, "budget_duration": "1d"}}),
+ (_TEAM_OPUS_GROUP_CAP, {"claude-opus-4-8": {"max_budget": 1000.0, "budget_duration": "1d"}}),
+ (
+ {"z-anthropic-group": {"max_budget": 10.0, "budget_duration": "1d"}},
+ {"claude-opus-4-8": {"max_budget": 1000.0, "budget_duration": "1d"}},
+ ),
+ (_TEAM_OPUS_FAMILY_CAP, {"claude-opus-4-8": {"max_budget": 1000.0, "budget_duration": "1d"}}),
+ (
+ {"anthropic/*": {"max_budget": 10.0, "budget_duration": "1d"}},
+ {"claude-opus-4-8": {"max_budget": 1000.0, "budget_duration": "1d"}},
+ ),
+ (
+ {"my-team/*": {"max_budget": 10.0, "budget_duration": "1d"}},
+ {"gpt-4o": {"max_budget": 1000.0, "budget_duration": "1d"}},
+ ),
+ (
+ {"team-route/*": {"max_budget": 10.0, "budget_duration": "1d"}},
+ {"gpt-4o": {"max_budget": 1000.0, "budget_duration": "1d"}},
+ ),
+ ],
+ ids=[
+ "group_over_pattern",
+ "group_rewindows_pattern",
+ "more_specific_pattern",
+ "earlier_group_sharing_a_model",
+ "model_inside_group",
+ "model_under_wildcard_group",
+ "model_matching_pattern",
+ "model_under_provider_pattern",
+ "model_under_served_prefix",
+ "model_under_wildcard_route",
+ ],
+)
+def test_team_admin_cannot_add_an_entry_looser_than_a_cap_it_outranks(
+ existing: Mapping[str, object], added: Mapping[str, object]
+) -> None:
+ with pytest.raises(HTTPException) as exc:
+ _check_team_admin_model_caps(existing=existing, requested={**existing, **added})
+ assert exc.value.status_code == 403
+ assert "proxy admin" in exc.value.detail["error"]
+ assert repr(next(iter(existing))) in exc.value.detail["error"]
+
+
+@pytest.mark.parametrize(
+ ("existing", "requested"),
+ [
+ (_TEAM_OPUS_FAMILY_CAP, {**_TEAM_OPUS_FAMILY_CAP, "opus-group": {"max_budget": 5.0, "budget_duration": "1d"}}),
+ (_TEAM_OPUS_FAMILY_CAP, {**_TEAM_OPUS_FAMILY_CAP, "claude*": {"max_budget": 1000.0, "budget_duration": "1d"}}),
+ (_TEAM_OPUS_GROUP_CAP, {**_TEAM_OPUS_GROUP_CAP, "a-gemini-group": {"max_budget": 1000.0, "budget_duration": "1d"}}),
+ (_TEAM_OPUS_GROUP_CAP, {**_TEAM_OPUS_GROUP_CAP, "zz-opus-group": {"max_budget": 1000.0, "budget_duration": "1d"}}),
+ (_TEAM_OPUS_GROUP_CAP, {**_TEAM_OPUS_GROUP_CAP, "gpt-4o": {"max_budget": 1000.0, "budget_duration": "1d"}}),
+ (_TEAM_OPUS_FAMILY_CAP, {**_TEAM_OPUS_FAMILY_CAP, "gpt-4o": {"max_budget": 1000.0, "budget_duration": "1d"}}),
+ (
+ _TEAM_OPUS_FAMILY_CAP,
+ {**_TEAM_OPUS_FAMILY_CAP, "a-gemini-group": {"max_budget": 1000.0, "budget_duration": "1d"}},
+ ),
+ (
+ {**_TEAM_OPUS_FAMILY_CAP, "opus-group": {"max_budget": 100.0, "budget_duration": "1d"}},
+ {**_TEAM_OPUS_FAMILY_CAP, "opus-group": {"max_budget": 100.0, "budget_duration": "1d"}},
+ ),
+ (
+ {**_TEAM_OPUS_FAMILY_CAP, "opus-group": {"max_budget": 100.0, "budget_duration": "1d"}},
+ {**_TEAM_OPUS_FAMILY_CAP, "opus-group": {"max_budget": 50.0, "budget_duration": "1d"}},
+ ),
+ (
+ {**_TEAM_OPUS_FAMILY_CAP, "opus-group": {"rpm_limit": 10}},
+ {**_TEAM_OPUS_FAMILY_CAP, "opus-group": {"rpm_limit": 20}},
+ ),
+ ],
+ ids=[
+ "tighter_group",
+ "less_specific_pattern",
+ "group_without_shared_models",
+ "later_group_sharing_a_model",
+ "model_outside_group",
+ "model_outside_pattern",
+ "group_outside_pattern",
+ "unchanged_looser_entry",
+ "tightened_looser_entry",
+ "uncapped_entry_edited",
+ ],
+)
+def test_team_admin_can_add_entries_that_never_lift_an_outranked_cap(
+ existing: Mapping[str, object], requested: Mapping[str, object]
+) -> None:
+ assert _check_team_admin_model_caps(existing=existing, requested=requested) is None
+
+
def test_team_model_cap_authority_skips_omitted_field_malformed_rows_and_proxy_admins() -> None:
from litellm.proxy.management_endpoints.team_endpoints import _check_team_model_budget_update_authority
diff --git a/tests/unit/test_internal_context.py b/tests/unit/test_internal_context.py
index f78c6064f76..3336cce9aaf 100644
--- a/tests/unit/test_internal_context.py
+++ b/tests/unit/test_internal_context.py
@@ -74,6 +74,7 @@ _IN_MEMORY_ONLY_CALLERS: Final = frozenset(
"litellm/proxy/container_endpoints/ownership.py",
"litellm/proxy/discovery_endpoints/agent_skills_endpoints.py",
"litellm/proxy/guardrails/guardrail_hooks/straiker/straiker.py",
+ "litellm/proxy/hooks/model_budget_shared_windows.py",
"litellm/proxy/spend_tracking/key_metadata_recovery.py",
"litellm/proxy/ui_crud_endpoints/latest_release_endpoints.py",
"litellm/responses/litellm_completion_transformation/transformation.py",
diff --git a/ui/litellm-dashboard/src/components/key_team_helpers/ModelMaxBudgetEditor.integration.test.tsx b/ui/litellm-dashboard/src/components/key_team_helpers/ModelMaxBudgetEditor.integration.test.tsx
index 33f09092cca..955b911ca97 100644
--- a/ui/litellm-dashboard/src/components/key_team_helpers/ModelMaxBudgetEditor.integration.test.tsx
+++ b/ui/litellm-dashboard/src/components/key_team_helpers/ModelMaxBudgetEditor.integration.test.tsx
@@ -1,5 +1,6 @@
+import userEvent from "@testing-library/user-event";
import { describe, expect, it, vi } from "vitest";
-import { renderWithProviders, screen } from "../../../tests/test-utils";
+import { fireEvent, renderWithProviders, screen } from "../../../tests/test-utils";
import { MODEL_MAX_BUDGET_PREMIUM_HINT, ModelMaxBudgetEditor, type ModelMaxBudget } from "./ModelMaxBudgetEditor";
const STORED: ModelMaxBudget = { "gpt-4o": { budget_limit: 5, time_period: "30d" } };
@@ -67,3 +68,65 @@ describe("ModelMaxBudgetEditor with an enterprise license", () => {
expect(addButton()).toBeEnabled();
});
});
+
+describe("ModelMaxBudgetEditor model families", () => {
+ it("stores a typed pattern that matches no listed model as one budget entry", async () => {
+ const user = userEvent.setup();
+ const onChange = vi.fn();
+ renderWithProviders(
+