mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
Merge 3d78d57662 into e0d1e64591
This commit is contained in:
commit
d89d7150a9
15 changed files with 1585 additions and 69 deletions
|
|
@ -434,6 +434,7 @@ _POSTGRES_OPERATION_BY_CALL_TYPE: Final[Mapping[str, PostgresOperation]] = Mappi
|
|||
"reset_budget_windows": PostgresOperation("select", None),
|
||||
"write_budget_windows": PostgresOperation("update", None),
|
||||
"roll_window_spend_row": PostgresOperation("update", None),
|
||||
"open_window_spend_row": PostgresOperation("upsert", None),
|
||||
"seed_window_spend": PostgresOperation("select", None),
|
||||
"select_window_spend_rows": PostgresOperation("select", None),
|
||||
"commit_window_spend_updates": PostgresOperation("upsert", None),
|
||||
|
|
|
|||
|
|
@ -64,6 +64,16 @@ _ROLL_WINDOW_SPEND_SQL: Final = (
|
|||
"AND window_start < ($4::timestamptz AT TIME ZONE 'UTC')"
|
||||
)
|
||||
|
||||
_OPEN_WINDOW_SPEND_SQL: Final = (
|
||||
'INSERT INTO "LiteLLM_BudgetWindowSpend" '
|
||||
"(entity_type, entity_id, window_duration, window_start, spend, created_at, updated_at) "
|
||||
"VALUES ($1, $2, $3, ($4::timestamptz AT TIME ZONE 'UTC'), 0, "
|
||||
"($4::timestamptz AT TIME ZONE 'UTC'), ($4::timestamptz AT TIME ZONE 'UTC')) "
|
||||
"ON CONFLICT (entity_type, entity_id, window_duration) DO UPDATE SET "
|
||||
"window_start = EXCLUDED.window_start, spend = 0, updated_at = EXCLUDED.updated_at "
|
||||
"WHERE \"LiteLLM_BudgetWindowSpend\".window_start <= ($5::timestamptz AT TIME ZONE 'UTC')"
|
||||
)
|
||||
|
||||
_SEED_FROM_SPEND_LOGS_KEY_SQL: Final = (
|
||||
"SELECT COALESCE(SUM(spend), 0.0) AS total, "
|
||||
"COALESCE(SUM(spend) FILTER (WHERE \"startTime\" < ($3::timestamptz AT TIME ZONE 'UTC')), 0.0) AS before_batch "
|
||||
|
|
@ -389,3 +399,29 @@ async def roll_window_spend_row(
|
|||
to_naive_utc(new_window_start),
|
||||
to_naive_utc(datetime.now(timezone.utc)),
|
||||
)
|
||||
|
||||
|
||||
async def open_window_spend_row(
|
||||
prisma_client: "PrismaClient",
|
||||
entity_type: str,
|
||||
entity_id: str,
|
||||
window_duration: str,
|
||||
window_start: datetime,
|
||||
replaces_window_start: datetime | None,
|
||||
) -> None:
|
||||
"""Start a window at `window_start` unless another writer already started one.
|
||||
|
||||
The row is created when it does not exist, and moved onto the new window only
|
||||
while it still holds `replaces_window_start`, the ended window the caller saw.
|
||||
A writer that lost the race leaves the winner's window in place, so reading
|
||||
the row back gives every writer the same window start.
|
||||
"""
|
||||
async with db_span("open_window_spend_row", "LiteLLM_BudgetWindowSpend"):
|
||||
await prisma_client.db.execute_raw(
|
||||
_OPEN_WINDOW_SPEND_SQL,
|
||||
entity_type,
|
||||
entity_id,
|
||||
window_duration,
|
||||
to_naive_utc(window_start),
|
||||
None if replaces_window_start is None else to_naive_utc(replaces_window_start),
|
||||
)
|
||||
|
|
|
|||
173
litellm/proxy/hooks/model_budget_shared_windows.py
Normal file
173
litellm/proxy/hooks/model_budget_shared_windows.py
Normal file
|
|
@ -0,0 +1,173 @@
|
|||
from collections.abc import Awaitable, Callable
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from typing import TYPE_CHECKING, Final, Protocol, TypeAlias
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.caching.in_memory_cache import InMemoryCache
|
||||
from litellm.proxy.db.budget_window_spend_writer import open_window_spend_row
|
||||
from litellm.proxy.db.db_transaction_queue.window_spend_update_queue import (
|
||||
WindowSpendTransaction,
|
||||
build_window_spend_transaction,
|
||||
)
|
||||
from litellm.repositories.table_repositories import BudgetWindowSpendRepository
|
||||
from litellm.router_strategy.budget_limiter import budget_window_seconds
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.proxy.utils import PrismaClient
|
||||
|
||||
MODEL_BUDGET_WINDOW_ENTITY_TYPE: Final = "model_max_budget"
|
||||
_READ_TTL_SECONDS: Final = 5
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SharedWindow:
|
||||
started_at: float
|
||||
spend: float
|
||||
|
||||
def is_open(self, budget_duration: str, now: float) -> bool:
|
||||
return now < self.started_at + budget_window_seconds(self.started_at, budget_duration)
|
||||
|
||||
|
||||
class LocalTTLCache(Protocol):
|
||||
def get_cache(self, key: str) -> object: ...
|
||||
|
||||
def set_cache(self, key: str, value: object, *, ttl: float) -> None: ...
|
||||
|
||||
|
||||
class SharedWindows(Protocol):
|
||||
async def read(self, spend_key: str, budget_duration: str) -> SharedWindow | None: ...
|
||||
|
||||
async def open(self, spend_key: str, budget_duration: str, now: float) -> float: ...
|
||||
|
||||
async def add(self, spend_key: str, budget_duration: str, started_at: float, spend: float) -> None: ...
|
||||
|
||||
|
||||
PrismaClientSource: TypeAlias = Callable[[], "PrismaClient | None"]
|
||||
WindowSpendSink: TypeAlias = Callable[[WindowSpendTransaction], Awaitable[None]]
|
||||
|
||||
|
||||
def _proxy_prisma_client() -> "PrismaClient | None":
|
||||
try:
|
||||
from litellm.proxy.proxy_server import prisma_client
|
||||
except ImportError:
|
||||
return None
|
||||
return prisma_client
|
||||
|
||||
|
||||
async def _queue_on_proxy_spend_writer(transaction: WindowSpendTransaction) -> None:
|
||||
from litellm.proxy.proxy_server import proxy_logging_obj
|
||||
|
||||
await proxy_logging_obj.db_spend_update_writer.window_spend_update_queue.add_update(transaction)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _Reading:
|
||||
window: SharedWindow | None
|
||||
|
||||
|
||||
class DatabaseSharedWindows:
|
||||
def __init__(
|
||||
self,
|
||||
prisma_client_source: PrismaClientSource = _proxy_prisma_client,
|
||||
sink: WindowSpendSink = _queue_on_proxy_spend_writer,
|
||||
read_ttl_seconds: int = _READ_TTL_SECONDS,
|
||||
) -> None:
|
||||
self._prisma_client_source = prisma_client_source
|
||||
self._sink = sink
|
||||
self._read_ttl_seconds = read_ttl_seconds
|
||||
self._readings: LocalTTLCache = InMemoryCache(max_size_in_memory=10_000, default_ttl=read_ttl_seconds)
|
||||
|
||||
async def read(self, spend_key: str, budget_duration: str) -> SharedWindow | None:
|
||||
cached: Final = self._readings.get_cache(key=spend_key)
|
||||
if isinstance(cached, _Reading):
|
||||
return cached.window
|
||||
window: Final = await self._read_row(spend_key=spend_key, budget_duration=budget_duration)
|
||||
self._readings.set_cache(key=spend_key, value=_Reading(window=window), ttl=self._read_ttl_seconds)
|
||||
return window
|
||||
|
||||
async def open(self, spend_key: str, budget_duration: str, now: float) -> float:
|
||||
"""When the window every replica shares started, opening one at `now` if none is open.
|
||||
|
||||
Falls back to `now`, this replica's own window, when the database cannot answer.
|
||||
"""
|
||||
prisma_client: Final = self._prisma_client_source()
|
||||
if prisma_client is None:
|
||||
return now
|
||||
try:
|
||||
seen: Final = await _fetch_row(
|
||||
prisma_client=prisma_client, spend_key=spend_key, budget_duration=budget_duration, use_writer=True
|
||||
)
|
||||
if seen is not None and seen.is_open(budget_duration=budget_duration, now=now):
|
||||
return seen.started_at
|
||||
await open_window_spend_row(
|
||||
prisma_client=prisma_client,
|
||||
entity_type=MODEL_BUDGET_WINDOW_ENTITY_TYPE,
|
||||
entity_id=spend_key,
|
||||
window_duration=budget_duration,
|
||||
window_start=datetime.fromtimestamp(now, tz=timezone.utc),
|
||||
replaces_window_start=None
|
||||
if seen is None
|
||||
else datetime.fromtimestamp(seen.started_at, tz=timezone.utc),
|
||||
)
|
||||
opened: Final = await _fetch_row(
|
||||
prisma_client=prisma_client, spend_key=spend_key, budget_duration=budget_duration, use_writer=True
|
||||
)
|
||||
except Exception: # noqa: BLE001 # an unreachable database leaves this replica on its own window
|
||||
verbose_proxy_logger.exception("Unable to open the shared model budget window")
|
||||
return now
|
||||
if opened is None or not opened.is_open(budget_duration=budget_duration, now=now):
|
||||
return now
|
||||
return opened.started_at
|
||||
|
||||
async def add(self, spend_key: str, budget_duration: str, started_at: float, spend: float) -> None:
|
||||
if self._prisma_client_source() is None:
|
||||
return
|
||||
try:
|
||||
await self._sink(
|
||||
build_window_spend_transaction(
|
||||
entity_type=MODEL_BUDGET_WINDOW_ENTITY_TYPE,
|
||||
entity_id=spend_key,
|
||||
window_duration=budget_duration,
|
||||
window_start=datetime.fromtimestamp(started_at, tz=timezone.utc),
|
||||
spend=spend,
|
||||
)
|
||||
)
|
||||
except Exception: # noqa: BLE001 # spend tracking must never fail the success callback
|
||||
verbose_proxy_logger.debug("Unable to queue shared model budget spend", exc_info=True)
|
||||
|
||||
async def _read_row(self, spend_key: str, budget_duration: str) -> SharedWindow | None:
|
||||
prisma_client: Final = self._prisma_client_source()
|
||||
if prisma_client is None:
|
||||
return None
|
||||
try:
|
||||
return await _fetch_row(
|
||||
prisma_client=prisma_client, spend_key=spend_key, budget_duration=budget_duration, use_writer=False
|
||||
)
|
||||
except Exception: # noqa: BLE001 # an unreadable row leaves enforcement on this replica's own counter
|
||||
verbose_proxy_logger.exception("Unable to read shared model budget spend")
|
||||
return None
|
||||
|
||||
|
||||
async def _fetch_row(
|
||||
prisma_client: "PrismaClient", spend_key: str, budget_duration: str, use_writer: bool
|
||||
) -> SharedWindow | None:
|
||||
row: Final = await BudgetWindowSpendRepository(prisma_client, use_writer=use_writer).table.find_unique(
|
||||
where={
|
||||
"entity_type_entity_id_window_duration": {
|
||||
"entity_type": MODEL_BUDGET_WINDOW_ENTITY_TYPE,
|
||||
"entity_id": spend_key,
|
||||
"window_duration": budget_duration,
|
||||
}
|
||||
}
|
||||
)
|
||||
if row is None:
|
||||
return None
|
||||
window_start: Final = row.window_start
|
||||
started_at: Final = (
|
||||
window_start if window_start.tzinfo is not None else window_start.replace(tzinfo=timezone.utc)
|
||||
).timestamp()
|
||||
return SharedWindow(started_at=started_at, spend=float(row.spend or 0.0))
|
||||
|
||||
|
||||
PROXY_SHARED_WINDOWS: Final = DatabaseSharedWindows()
|
||||
|
|
@ -1,11 +1,18 @@
|
|||
import asyncio
|
||||
import functools
|
||||
import itertools
|
||||
import json
|
||||
import time
|
||||
from collections.abc import Callable, Iterable, Mapping, Sequence
|
||||
from collections.abc import Callable, Collection, Iterable, Iterator, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from types import MappingProxyType
|
||||
from typing import Final, cast # noqa: TID251 # untyped callback payload requires explicit narrowing
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Final,
|
||||
TypeAlias,
|
||||
cast, # noqa: TID251 # untyped callback payload requires explicit narrowing
|
||||
)
|
||||
|
||||
import litellm
|
||||
from litellm._internal_context import with_service_target
|
||||
|
|
@ -16,10 +23,19 @@ from litellm.litellm_core_utils.duration_parser import duration_in_seconds
|
|||
from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
|
||||
from litellm.llms.bedrock.common_utils import get_bedrock_base_model
|
||||
from litellm.proxy._types import Litellm_EntityType, UserAPIKeyAuth
|
||||
from litellm.proxy.hooks.model_budget_shared_windows import (
|
||||
PROXY_SHARED_WINDOWS,
|
||||
LocalTTLCache,
|
||||
SharedWindow,
|
||||
SharedWindows,
|
||||
)
|
||||
from litellm.router_strategy.budget_limiter import RouterBudgetLimiting, budget_window_seconds
|
||||
from litellm.router_utils.batch_utils import batch_charged_once_marker_key, batch_id_to_charge_once
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.types.utils import BudgetConfig, StandardLoggingPayload
|
||||
from litellm.types.utils import BudgetConfig, LlmProviders, StandardLoggingPayload
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.router import Router
|
||||
|
||||
VIRTUAL_KEY_SPEND_CACHE_KEY_PREFIX: Final = "virtual_key_spend"
|
||||
END_USER_SPEND_CACHE_KEY_PREFIX: Final = "end_user_model_spend"
|
||||
|
|
@ -39,10 +55,18 @@ _SPEND_CACHE_KEY_PREFIXES: Final = MappingProxyType(
|
|||
|
||||
_LEGACY_REQUEST_MODEL_SCOPES: Final = frozenset({Litellm_EntityType.KEY, Litellm_EntityType.END_USER})
|
||||
|
||||
_PER_REPLICA_SCOPES: Final = frozenset({Litellm_EntityType.END_USER})
|
||||
|
||||
_PROCESS_STARTED_AT: Final = time.monotonic()
|
||||
|
||||
_LATEST_RESET_AT: Final = datetime(9999, 12, 31, tzinfo=timezone.utc).timestamp()
|
||||
|
||||
_GLOB_WILDCARD: Final = "*"
|
||||
|
||||
ModelAccessGroups: TypeAlias = Callable[[str], Collection[str]]
|
||||
|
||||
_PROVIDER_PREFIXES: Final = tuple(f"{provider.value}/" for provider in LlmProviders)
|
||||
|
||||
_BUDGET_START_TIME_KEY_PREFIXES: Final = MappingProxyType(
|
||||
{
|
||||
Litellm_EntityType.KEY: "virtual_key_budget_start_time",
|
||||
|
|
@ -68,6 +92,15 @@ class ResolvedModelBudget:
|
|||
budget_config: BudgetConfig
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class RoutedModels:
|
||||
served_names: frozenset[str]
|
||||
group_members: Mapping[str, frozenset[str]]
|
||||
|
||||
def members(self, access_group: str) -> frozenset[str]:
|
||||
return self.group_members.get(access_group, frozenset())
|
||||
|
||||
|
||||
def team_member_budget_entity_id(user_id: str, team_id: str) -> str:
|
||||
return f"{len(user_id)}:{user_id}:{team_id}"
|
||||
|
||||
|
|
@ -131,36 +164,219 @@ def model_budget_start_time_cache_key(
|
|||
return f"{_BUDGET_START_TIME_KEY_PREFIXES[entity_type]}:{entity_id}:{budget_model}:{budget_duration}"
|
||||
|
||||
|
||||
def resolve_model_budget(model: str, model_max_budget: Mapping[str, object]) -> ResolvedModelBudget | None:
|
||||
"""Find the `model_max_budget` entry that governs `model`, or None."""
|
||||
for candidate in _budget_model_candidates(model):
|
||||
raw_budget_config = model_max_budget.get(candidate)
|
||||
if raw_budget_config is None:
|
||||
continue
|
||||
if (budget_config := _usable_budget_config(raw_budget_config)) is None:
|
||||
# An entry that will not validate cannot be keyed, so it cannot be
|
||||
# enforced or incremented. Skip to the next candidate rather than
|
||||
# raising: raising would abort every other scope's increment and turn
|
||||
# a config typo into a 500, and stopping here would let one malformed
|
||||
# specific entry disable a perfectly good bare-family budget beside
|
||||
# it. The candidate chain already falls through an ABSENT entry, and
|
||||
# an unparseable one is indistinguishable from absent to enforcement.
|
||||
# `validate_model_max_budget` rejects these on the write path, so
|
||||
# reaching here means config.yaml or a direct DB edit.
|
||||
verbose_proxy_logger.warning(
|
||||
"Ignoring unusable model_max_budget entry for %s; it cannot be enforced or tracked",
|
||||
candidate,
|
||||
)
|
||||
continue
|
||||
return ResolvedModelBudget(budget_model=candidate, budget_config=budget_config)
|
||||
return None
|
||||
def model_access_groups_of(router: "Router", model: str) -> frozenset[str]:
|
||||
"""Access groups serving `model`, leaving out any named after a model the router serves.
|
||||
|
||||
An entry with that name already budgets that model, so reading it as the group too would quietly widen it.
|
||||
"""
|
||||
return frozenset(
|
||||
group for group in router.get_model_access_groups(model_name=model) if group not in router.model_names
|
||||
)
|
||||
|
||||
|
||||
def routed_models_of(router: "Router") -> RoutedModels:
|
||||
served_names: Final = frozenset(router.model_names)
|
||||
return RoutedModels(
|
||||
served_names=served_names,
|
||||
group_members=MappingProxyType(
|
||||
{
|
||||
group: frozenset(members)
|
||||
for group, members in router.get_model_access_groups().items()
|
||||
if group not in served_names
|
||||
}
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def proxy_model_access_groups(model: str) -> frozenset[str]:
|
||||
router: Final = _proxy_router()
|
||||
return frozenset() if router is None else model_access_groups_of(router=router, model=model)
|
||||
|
||||
|
||||
def proxy_routed_models() -> RoutedModels:
|
||||
router: Final = _proxy_router()
|
||||
if router is None:
|
||||
return RoutedModels(served_names=frozenset(), group_members=MappingProxyType({}))
|
||||
return routed_models_of(router)
|
||||
|
||||
|
||||
def _proxy_router() -> "Router | None":
|
||||
try:
|
||||
from litellm.proxy.proxy_server import llm_router
|
||||
except ImportError:
|
||||
return None
|
||||
return llm_router
|
||||
|
||||
|
||||
def resolve_model_budget(
|
||||
model: str,
|
||||
model_max_budget: Mapping[str, object],
|
||||
model_access_groups: ModelAccessGroups = proxy_model_access_groups,
|
||||
) -> ResolvedModelBudget | None:
|
||||
"""Find the `model_max_budget` entry that governs `model`, or None.
|
||||
|
||||
An entry named after the model wins, then one named after a model access
|
||||
group serving it, then the most specific glob (`claude-opus*`). Every model
|
||||
an access group or glob entry covers draws from that entry's one counter.
|
||||
"""
|
||||
resolved: Final = (
|
||||
_usable_entry(entry=entry, raw_budget_config=model_max_budget[entry])
|
||||
for entry in _governing_entry_names(
|
||||
model=model, model_max_budget=model_max_budget, model_access_groups=model_access_groups
|
||||
)
|
||||
)
|
||||
return next((entry for entry in resolved if entry is not None), None)
|
||||
|
||||
|
||||
def _governing_entry_names(
|
||||
model: str,
|
||||
model_max_budget: Mapping[str, object],
|
||||
model_access_groups: ModelAccessGroups,
|
||||
) -> Iterator[str]:
|
||||
candidates: Final = _budget_model_candidates(model)
|
||||
yield from (candidate for candidate in candidates if candidate in model_max_budget)
|
||||
group_entries: Final = tuple(
|
||||
sorted(entry for entry in model_max_budget if _GLOB_WILDCARD not in entry and entry not in candidates)
|
||||
)
|
||||
if group_entries:
|
||||
groups: Final = model_access_groups(model)
|
||||
yield from (entry for entry in group_entries if entry in groups)
|
||||
yield from sorted(
|
||||
(entry for entry in model_max_budget if _GLOB_WILDCARD in entry and _glob_matches(entry, candidates)),
|
||||
key=_glob_precedence,
|
||||
reverse=True,
|
||||
)
|
||||
|
||||
|
||||
def _glob_matches(pattern: str, names: Iterable[str]) -> bool:
|
||||
return any(_glob_matches_name(pattern=pattern, name=name) for name in names)
|
||||
|
||||
|
||||
def _glob_matches_name(pattern: str, name: str) -> bool:
|
||||
"""Whole-name match where `*` is the only wildcard, in linear time whatever the pattern holds.
|
||||
|
||||
Taking each literal piece between two wildcards at its first occurrence is
|
||||
always safe, since a later occurrence only leaves less room for the rest, so
|
||||
the match never needs to backtrack the way a `.*` regex does.
|
||||
"""
|
||||
head, *middle, tail = pattern.split(_GLOB_WILDCARD)
|
||||
middle_end: Final = len(name) - len(tail)
|
||||
if middle_end < len(head) or not name.startswith(head) or not name.endswith(tail):
|
||||
return False
|
||||
return functools.reduce(_after_piece(name=name, end=middle_end), middle, len(head)) >= 0
|
||||
|
||||
|
||||
def _after_piece(name: str, end: int) -> Callable[[int, str], int]:
|
||||
def step(position: int, piece: str) -> int:
|
||||
found: Final = name.find(piece, position, end) if position >= 0 else -1
|
||||
return found + len(piece) if found >= 0 else -1
|
||||
|
||||
return step
|
||||
|
||||
|
||||
def _glob_precedence(pattern: str) -> tuple[int, int, str]:
|
||||
"""More literal characters first, then fewer wildcards, so padding a pattern with `*` never makes it win."""
|
||||
wildcards: Final = pattern.count(_GLOB_WILDCARD)
|
||||
return len(pattern) - wildcards, -wildcards, pattern
|
||||
|
||||
|
||||
def entries_outranked_by(entry: str, model_max_budget: Collection[str], routed_models: RoutedModels) -> Iterator[str]:
|
||||
"""Entries that `entry` can take precedence over for some model a request can name.
|
||||
|
||||
Two patterns can both match a name the router has not served yet, so a
|
||||
pattern outranks every less specific pattern. A model or group entry
|
||||
outranks a pattern that matches one of the names it governs: the model,
|
||||
the model under any provider prefix or wildcard route, a served name
|
||||
ending in it, or a member of the group, and every pattern when a member is
|
||||
itself a wildcard route.
|
||||
"""
|
||||
if _GLOB_WILDCARD in entry:
|
||||
entry_precedence: Final = _glob_precedence(entry)
|
||||
yield from (
|
||||
other
|
||||
for other in model_max_budget
|
||||
if _GLOB_WILDCARD in other and _glob_precedence(other) < entry_precedence
|
||||
)
|
||||
return
|
||||
entry_members: Final = routed_models.members(entry)
|
||||
covers_wildcard_route: Final = any(_GLOB_WILDCARD in member for member in entry_members)
|
||||
governed_candidates: Final = frozenset(
|
||||
itertools.chain.from_iterable(
|
||||
_budget_model_candidates(name)
|
||||
for name in _names_governed_by(entry=entry, entry_members=entry_members, routed_models=routed_models)
|
||||
)
|
||||
)
|
||||
yield from (
|
||||
other
|
||||
for other in model_max_budget
|
||||
if _GLOB_WILDCARD in other and (covers_wildcard_route or _glob_matches(other, governed_candidates))
|
||||
)
|
||||
yield from (
|
||||
other
|
||||
for other in model_max_budget
|
||||
if _GLOB_WILDCARD not in other
|
||||
and _ranks_ahead_of_group(
|
||||
entry=entry, entry_members=entry_members, group=other, group_members=routed_models.members(other)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _names_governed_by(entry: str, entry_members: frozenset[str], routed_models: RoutedModels) -> Iterator[str]:
|
||||
yield entry
|
||||
yield from (f"{prefix}{entry}" for prefix in _PROVIDER_PREFIXES)
|
||||
yield from (
|
||||
name
|
||||
for name in routed_models.served_names
|
||||
if _GLOB_WILDCARD not in name and entry in _budget_model_candidates(name)
|
||||
)
|
||||
yield from (name.replace(_GLOB_WILDCARD, entry) for name in routed_models.served_names if _GLOB_WILDCARD in name)
|
||||
yield from (member for member in entry_members if _GLOB_WILDCARD not in member)
|
||||
|
||||
|
||||
def _ranks_ahead_of_group(entry: str, entry_members: frozenset[str], group: str, group_members: Iterable[str]) -> bool:
|
||||
return any(
|
||||
_GLOB_WILDCARD in member
|
||||
or entry in _budget_model_candidates(member)
|
||||
or (member in entry_members and entry < group)
|
||||
for member in group_members
|
||||
)
|
||||
|
||||
|
||||
def _usable_entry(entry: str, raw_budget_config: object) -> ResolvedModelBudget | None:
|
||||
if (budget_config := _usable_budget_config(raw_budget_config)) is None:
|
||||
# An entry that will not validate cannot be keyed, so it cannot be
|
||||
# enforced or incremented. Skip to the next candidate rather than
|
||||
# raising: raising would abort every other scope's increment and turn
|
||||
# a config typo into a 500, and stopping here would let one malformed
|
||||
# specific entry disable a perfectly good bare-family budget beside
|
||||
# it. The candidate chain already falls through an ABSENT entry, and
|
||||
# an unparseable one is indistinguishable from absent to enforcement.
|
||||
# `validate_model_max_budget` rejects these on the write path, so
|
||||
# reaching here means config.yaml or a direct DB edit.
|
||||
verbose_proxy_logger.warning(
|
||||
"Ignoring unusable model_max_budget entry for %s; it cannot be enforced or tracked",
|
||||
entry,
|
||||
)
|
||||
return None
|
||||
return ResolvedModelBudget(budget_model=entry, budget_config=budget_config)
|
||||
|
||||
|
||||
def team_model_budget_applies(model: str, key_model_max_budget: Mapping[str, object] | None) -> bool:
|
||||
"""A key entry that spend-gates `model` overrides the team cap: it is then gated on and billed to the key alone."""
|
||||
"""A key entry named after `model` that spend-gates it overrides the team cap: it is then gated on and billed to
|
||||
the key alone. A key pattern or group entry is gated alongside the team cap, never instead of it."""
|
||||
if not key_model_max_budget:
|
||||
return True
|
||||
resolved: Final = resolve_model_budget(model=model, model_max_budget=key_model_max_budget)
|
||||
resolved: Final = resolve_model_budget(
|
||||
model=model,
|
||||
model_max_budget=MappingProxyType(
|
||||
{
|
||||
candidate: key_model_max_budget[candidate]
|
||||
for candidate in _budget_model_candidates(model)
|
||||
if candidate in key_model_max_budget
|
||||
}
|
||||
),
|
||||
model_access_groups=lambda _: frozenset(),
|
||||
)
|
||||
return resolved is None or not _spend_gated(resolved.budget_config)
|
||||
|
||||
|
||||
|
|
@ -208,6 +424,7 @@ async def build_model_max_budget_usage(
|
|||
model_max_budget: Mapping[str, object] | None,
|
||||
cache: DualCache | None,
|
||||
now: datetime | None = None,
|
||||
shared_windows: SharedWindows = PROXY_SHARED_WINDOWS,
|
||||
) -> dict[str, dict[str, object]]:
|
||||
"""Current-window spend per configured budget model, as `/key/info` and `/user/info` report it.
|
||||
|
||||
|
|
@ -231,17 +448,18 @@ async def build_model_max_budget_usage(
|
|||
)
|
||||
if not budgets:
|
||||
return {}
|
||||
spend_keys: Final = tuple(
|
||||
model_budget_spend_cache_key(
|
||||
entity_type=entity_type,
|
||||
entity_id=entity_id,
|
||||
budget_model=budget_model,
|
||||
budget_duration=budget_config.budget_duration,
|
||||
)
|
||||
for budget_model, budget_config in budgets
|
||||
)
|
||||
windows: Final = await _read_windows(
|
||||
cache=cache,
|
||||
spend_keys=tuple(
|
||||
model_budget_spend_cache_key(
|
||||
entity_type=entity_type,
|
||||
entity_id=entity_id,
|
||||
budget_model=budget_model,
|
||||
budget_duration=budget_config.budget_duration,
|
||||
)
|
||||
for budget_model, budget_config in budgets
|
||||
),
|
||||
spend_keys=spend_keys,
|
||||
start_time_keys=tuple(
|
||||
model_budget_start_time_cache_key(
|
||||
entity_type=entity_type,
|
||||
|
|
@ -253,21 +471,57 @@ async def build_model_max_budget_usage(
|
|||
),
|
||||
)
|
||||
current_time: Final = (now or datetime.now(timezone.utc)).timestamp()
|
||||
shared: Final = tuple(
|
||||
[
|
||||
(
|
||||
await _open_shared_window(
|
||||
shared_windows=shared_windows,
|
||||
spend_key=spend_key,
|
||||
budget_duration=budget_config.budget_duration,
|
||||
now=current_time,
|
||||
)
|
||||
if cache.redis_cache is None
|
||||
else None
|
||||
)
|
||||
for spend_key, (_, budget_config) in zip(spend_keys, budgets, strict=True)
|
||||
]
|
||||
)
|
||||
readings: Final = tuple(
|
||||
_fuller_reading(local=window, shared=shared_window)
|
||||
for window, shared_window in zip(windows, shared, strict=True)
|
||||
)
|
||||
return {
|
||||
budget_model: {
|
||||
"current_spend": round(window.current_spend, 4),
|
||||
"current_spend": round(reading.current_spend, 4),
|
||||
"budget_limit": budget_config.max_budget,
|
||||
"time_period": budget_config.budget_duration,
|
||||
"budget_reset_at": _window_reset_at(
|
||||
started_at=window.started_at,
|
||||
started_at=reading.started_at,
|
||||
budget_duration=budget_config.budget_duration,
|
||||
now=current_time,
|
||||
),
|
||||
}
|
||||
for (budget_model, budget_config), window in zip(budgets, windows, strict=True)
|
||||
for (budget_model, budget_config), reading in zip(budgets, readings, strict=True)
|
||||
}
|
||||
|
||||
|
||||
async def _open_shared_window(
|
||||
shared_windows: SharedWindows, spend_key: str, budget_duration: str | None, now: float
|
||||
) -> SharedWindow | None:
|
||||
if budget_duration is None:
|
||||
return None
|
||||
window: Final = await shared_windows.read(spend_key=spend_key, budget_duration=budget_duration)
|
||||
if window is None or not window.is_open(budget_duration=budget_duration, now=now):
|
||||
return None
|
||||
return window
|
||||
|
||||
|
||||
def _fuller_reading(local: _WindowReading, shared: SharedWindow | None) -> _WindowReading:
|
||||
if shared is None or shared.spend <= local.current_spend:
|
||||
return local
|
||||
return _WindowReading(current_spend=shared.spend, started_at=shared.started_at)
|
||||
|
||||
|
||||
@with_service_target("model_budgets")
|
||||
async def _read_windows(
|
||||
cache: DualCache, spend_keys: Sequence[str], start_time_keys: Sequence[str]
|
||||
|
|
@ -337,13 +591,18 @@ def _as_float(value: object) -> float | None:
|
|||
def _resolve_entity_model_budgets(
|
||||
model: str,
|
||||
entity_budgets: Iterable[tuple[Litellm_EntityType, str | None, object]],
|
||||
model_access_groups: ModelAccessGroups,
|
||||
) -> tuple[tuple[Litellm_EntityType, str, ResolvedModelBudget], ...]:
|
||||
"""Drop the scopes that do not budget `model`, keeping only what can be incremented."""
|
||||
return tuple(
|
||||
(entity_type, entity_id, resolved)
|
||||
for entity_type, entity_id, model_max_budget in entity_budgets
|
||||
if entity_id is not None and isinstance(model_max_budget, Mapping) and model_max_budget
|
||||
for resolved in (resolve_model_budget(model=model, model_max_budget=model_max_budget),)
|
||||
for resolved in (
|
||||
resolve_model_budget(
|
||||
model=model, model_max_budget=model_max_budget, model_access_groups=model_access_groups
|
||||
),
|
||||
)
|
||||
if resolved is not None and resolved.budget_config.budget_duration is not None
|
||||
)
|
||||
|
||||
|
|
@ -367,9 +626,17 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
|
|||
Example: key=$LITELLM_API_KEY, model=gpt-4o, max_budget=100, time_period=1d
|
||||
"""
|
||||
|
||||
def __init__(self, dual_cache: DualCache, clock: Callable[[], float] = time.time):
|
||||
def __init__(
|
||||
self,
|
||||
dual_cache: DualCache,
|
||||
clock: Callable[[], float] = time.time,
|
||||
model_access_groups: ModelAccessGroups = proxy_model_access_groups,
|
||||
shared_windows: SharedWindows = PROXY_SHARED_WINDOWS,
|
||||
):
|
||||
self.dual_cache = dual_cache
|
||||
self._clock = clock
|
||||
self._model_access_groups = model_access_groups
|
||||
self._shared_windows = shared_windows
|
||||
self.redis_increment_operation_queue = []
|
||||
self._redis_increment_queue_lock = asyncio.Lock()
|
||||
self._redis_increment_flush_lock = asyncio.Lock()
|
||||
|
|
@ -511,7 +778,9 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
|
|||
) -> bool:
|
||||
if not model_max_budget:
|
||||
return True
|
||||
resolved: Final = resolve_model_budget(model=model, model_max_budget=model_max_budget)
|
||||
resolved: Final = resolve_model_budget(
|
||||
model=model, model_max_budget=model_max_budget, model_access_groups=self._model_access_groups
|
||||
)
|
||||
if resolved is None:
|
||||
verbose_proxy_logger.debug("Model %s not found in %s model_max_budget", model, entity_type.value)
|
||||
return True
|
||||
|
|
@ -520,15 +789,26 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
|
|||
if max_budget is None or max_budget < 0:
|
||||
return True
|
||||
|
||||
current_spend: Final = await self._get_spend_for_model_budget(
|
||||
local_spend: Final = await self._get_spend_for_model_budget(
|
||||
entity_type=entity_type,
|
||||
entity_id=entity_id,
|
||||
model=model,
|
||||
resolved=resolved,
|
||||
)
|
||||
current_spend: Final = (
|
||||
local_spend
|
||||
if local_spend >= max_budget
|
||||
else await self._with_shared_spend(
|
||||
entity_type=entity_type, entity_id=entity_id, resolved=resolved, local_spend=local_spend
|
||||
)
|
||||
)
|
||||
if current_spend >= max_budget:
|
||||
raise litellm.BudgetExceededError(
|
||||
message=exceeded_message,
|
||||
message=(
|
||||
exceeded_message
|
||||
if resolved.budget_model in _budget_model_candidates(model)
|
||||
else f"{exceeded_message}, budget_model={resolved.budget_model}"
|
||||
),
|
||||
current_cost=current_spend,
|
||||
max_budget=max_budget,
|
||||
entity_type=entity_type.value,
|
||||
|
|
@ -566,6 +846,30 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
|
|||
return current_spend
|
||||
return current_spend + _as_spend(await self._cached_spend(legacy_spend_key))
|
||||
|
||||
async def _with_shared_spend(
|
||||
self,
|
||||
entity_type: Litellm_EntityType,
|
||||
entity_id: str | None,
|
||||
resolved: ResolvedModelBudget,
|
||||
local_spend: float,
|
||||
) -> float:
|
||||
"""Without Redis each replica's counter holds only its own share, so the window every replica adds to
|
||||
in the database is the floor."""
|
||||
if not self._shares_windows_in_database(entity_type):
|
||||
return local_spend
|
||||
shared: Final = await _open_shared_window(
|
||||
shared_windows=self._shared_windows,
|
||||
spend_key=model_budget_spend_cache_key(
|
||||
entity_type=entity_type,
|
||||
entity_id=entity_id,
|
||||
budget_model=resolved.budget_model,
|
||||
budget_duration=resolved.budget_config.budget_duration,
|
||||
),
|
||||
budget_duration=resolved.budget_config.budget_duration,
|
||||
now=self._clock(),
|
||||
)
|
||||
return local_spend if shared is None else max(local_spend, shared.spend)
|
||||
|
||||
async def _cached_spend(self, spend_key: str) -> float | None:
|
||||
redis_cache: Final = self.dual_cache.redis_cache
|
||||
if redis_cache is None:
|
||||
|
|
@ -669,7 +973,9 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
|
|||
),
|
||||
)
|
||||
|
||||
resolved_budgets: Final = _resolve_entity_model_budgets(model=model, entity_budgets=entity_budgets)
|
||||
resolved_budgets: Final = _resolve_entity_model_budgets(
|
||||
model=model, entity_budgets=entity_budgets, model_access_groups=self._model_access_groups
|
||||
)
|
||||
if not resolved_budgets:
|
||||
verbose_proxy_logger.debug(
|
||||
"Not running _PROXY_VirtualKeyModelMaxBudgetLimiter.async_log_success_event: "
|
||||
|
|
@ -723,17 +1029,78 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
|
|||
ttl_seconds=duration_in_seconds(budget_duration),
|
||||
):
|
||||
return
|
||||
start_time_key: Final = model_budget_start_time_cache_key(
|
||||
entity_type=entity_type,
|
||||
entity_id=entity_id,
|
||||
budget_model=resolved.budget_model,
|
||||
budget_duration=budget_duration,
|
||||
)
|
||||
shares_window: Final = self._shares_windows_in_database(entity_type)
|
||||
if shares_window:
|
||||
await self._join_shared_window(
|
||||
spend_key=spend_key, start_time_key=start_time_key, budget_duration=budget_duration
|
||||
)
|
||||
await self._increment_spend_for_key(
|
||||
budget_config=resolved.budget_config,
|
||||
spend_key=spend_key,
|
||||
start_time_key=model_budget_start_time_cache_key(
|
||||
entity_type=entity_type,
|
||||
entity_id=entity_id,
|
||||
budget_model=resolved.budget_model,
|
||||
budget_duration=budget_duration,
|
||||
),
|
||||
start_time_key=start_time_key,
|
||||
response_cost=response_cost,
|
||||
)
|
||||
if shares_window:
|
||||
await self._share_spend(
|
||||
spend_key=spend_key,
|
||||
start_time_key=start_time_key,
|
||||
budget_duration=budget_duration,
|
||||
response_cost=response_cost,
|
||||
)
|
||||
|
||||
def _shares_windows_in_database(self, entity_type: Litellm_EntityType) -> bool:
|
||||
"""End-user ids are whatever the client sends, so their windows stay on each replica rather than adding a
|
||||
database row per id."""
|
||||
return self.dual_cache.redis_cache is None and entity_type not in _PER_REPLICA_SCOPES
|
||||
|
||||
async def _join_shared_window(self, spend_key: str, start_time_key: str, budget_duration: str) -> None:
|
||||
"""Without Redis, start this replica's window where the database says every replica's window starts.
|
||||
|
||||
Every replica then sends the same window start with its spend, so no flush reads as a newer window that
|
||||
replaces the spend the others already added.
|
||||
"""
|
||||
now: Final = self._clock()
|
||||
local: Final = self.dual_cache.in_memory_cache
|
||||
if _open_local_window_start(
|
||||
local=local, start_time_key=start_time_key, budget_duration=budget_duration, now=now
|
||||
):
|
||||
return
|
||||
started_at: Final = await self._shared_windows.open(
|
||||
spend_key=spend_key, budget_duration=budget_duration, now=now
|
||||
)
|
||||
if _open_local_window_start(
|
||||
local=local, start_time_key=start_time_key, budget_duration=budget_duration, now=now
|
||||
):
|
||||
return
|
||||
_start_local_window(
|
||||
local=local,
|
||||
spend_key=spend_key,
|
||||
start_time_key=start_time_key,
|
||||
started_at=started_at,
|
||||
budget_duration=budget_duration,
|
||||
now=now,
|
||||
)
|
||||
|
||||
async def _share_spend(
|
||||
self, spend_key: str, start_time_key: str, budget_duration: str, response_cost: float
|
||||
) -> None:
|
||||
now: Final = self._clock()
|
||||
await self._shared_windows.add(
|
||||
spend_key=spend_key,
|
||||
budget_duration=budget_duration,
|
||||
started_at=await self._get_or_set_budget_start_time(
|
||||
start_time_key=start_time_key,
|
||||
current_time=now,
|
||||
ttl_seconds=budget_window_seconds(now, budget_duration),
|
||||
),
|
||||
spend=response_cost,
|
||||
)
|
||||
|
||||
async def _claim_batch_charge(self, spend_key: str, batch_id: str, ttl_seconds: int) -> bool:
|
||||
marker_key: Final = batch_charged_once_marker_key(spend_key=spend_key, batch_id=batch_id)
|
||||
|
|
@ -743,4 +1110,17 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
|
|||
return polls == 1
|
||||
|
||||
|
||||
def _open_local_window_start(local: LocalTTLCache, start_time_key: str, budget_duration: str, now: float) -> bool:
|
||||
started_at: Final = _as_float(local.get_cache(key=start_time_key))
|
||||
return started_at is not None and now - started_at <= budget_window_seconds(started_at, budget_duration)
|
||||
|
||||
|
||||
def _start_local_window(
|
||||
local: LocalTTLCache, spend_key: str, start_time_key: str, started_at: float, budget_duration: str, now: float
|
||||
) -> None:
|
||||
ttl: Final = started_at + budget_window_seconds(started_at, budget_duration) - now
|
||||
local.set_cache(key=spend_key, value=0.0, ttl=ttl)
|
||||
local.set_cache(key=start_time_key, value=started_at, ttl=ttl)
|
||||
|
||||
|
||||
PROXY_VirtualKeyModelMaxBudgetLimiter = _PROXY_VirtualKeyModelMaxBudgetLimiter
|
||||
|
|
|
|||
|
|
@ -139,7 +139,10 @@ from litellm.proxy.common_utils.user_api_key_cache import (
|
|||
from litellm.proxy.db.db_span import db_span
|
||||
from litellm.proxy.hooks.key_management_event_hooks import KeyManagementEventHooks
|
||||
from litellm.proxy.hooks.model_max_budget_limiter import (
|
||||
RoutedModels,
|
||||
build_model_max_budget_usage,
|
||||
entries_outranked_by,
|
||||
proxy_routed_models,
|
||||
resolve_model_budget,
|
||||
)
|
||||
from litellm.proxy.management.teams.authz import TEAM_OR_ORG_ADMIN, TeamRole, is_team_admin, team_access_denied
|
||||
|
|
@ -1467,6 +1470,7 @@ def _check_team_model_budget_update_authority(
|
|||
data: UpdateTeamRequest,
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
existing_model_max_budget: Mapping[str, object] | None,
|
||||
routed_models_source: Callable[[], RoutedModels] = proxy_routed_models,
|
||||
) -> None:
|
||||
"""Like `_check_team_budget_update_authority`: only a proxy admin may raise, re-window or drop a per-model cap."""
|
||||
if user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN:
|
||||
|
|
@ -1494,11 +1498,7 @@ def _check_team_model_budget_update_authority(
|
|||
cap = governing.budget_config
|
||||
if cap.max_budget is None:
|
||||
continue
|
||||
if (
|
||||
proposed.max_budget is None
|
||||
or proposed.max_budget > cap.max_budget
|
||||
or proposed.budget_duration != cap.budget_duration
|
||||
):
|
||||
if _looser_model_cap(proposed=proposed, cap=cap):
|
||||
raise HTTPException(
|
||||
status_code=403,
|
||||
detail={
|
||||
|
|
@ -1510,6 +1510,74 @@ def _check_team_model_budget_update_authority(
|
|||
)
|
||||
},
|
||||
)
|
||||
routed_models: Final = routed_models_source()
|
||||
for model_name, proposed in requested.items():
|
||||
_check_outranked_model_caps(
|
||||
entry=model_name,
|
||||
proposed=proposed,
|
||||
existing_model_max_budget=existing_model_max_budget,
|
||||
routed_models=routed_models,
|
||||
)
|
||||
|
||||
|
||||
def _check_outranked_model_caps(
|
||||
entry: str,
|
||||
proposed: BudgetConfig,
|
||||
existing_model_max_budget: Mapping[str, object],
|
||||
routed_models: RoutedModels,
|
||||
) -> None:
|
||||
if _keeps_or_tightens(proposed=proposed, existing=_existing_model_cap(existing_model_max_budget.get(entry))):
|
||||
return
|
||||
outranked: Final = next(
|
||||
_looser_than_outranked_caps(entry, proposed, existing_model_max_budget, routed_models), None
|
||||
)
|
||||
if outranked is None:
|
||||
return
|
||||
outranked_name, cap = outranked
|
||||
raise HTTPException(
|
||||
status_code=403,
|
||||
detail={
|
||||
"error": (
|
||||
f"Only a proxy admin can set a team's model_max_budget entry {entry!r} looser than "
|
||||
f"{outranked_name!r}, which it can take precedence over. Current max_budget={cap.max_budget} per "
|
||||
f"{cap.budget_duration} (entry {outranked_name!r}), requested={proposed.max_budget} per "
|
||||
f"{proposed.budget_duration}."
|
||||
)
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _looser_than_outranked_caps(
|
||||
entry: str,
|
||||
proposed: BudgetConfig,
|
||||
existing_model_max_budget: Mapping[str, object],
|
||||
routed_models: RoutedModels,
|
||||
) -> Iterator[tuple[str, BudgetConfig]]:
|
||||
outranked_caps: Final = (
|
||||
(outranked, _existing_model_cap(existing_model_max_budget[outranked]))
|
||||
for outranked in entries_outranked_by(
|
||||
entry=entry, model_max_budget=existing_model_max_budget, routed_models=routed_models
|
||||
)
|
||||
)
|
||||
yield from (
|
||||
(outranked, cap)
|
||||
for outranked, cap in outranked_caps
|
||||
if cap is not None and cap.max_budget is not None and _looser_model_cap(proposed=proposed, cap=cap)
|
||||
)
|
||||
|
||||
|
||||
def _keeps_or_tightens(proposed: BudgetConfig, existing: BudgetConfig | None) -> bool:
|
||||
return existing is not None and (
|
||||
existing.max_budget is None or not _looser_model_cap(proposed=proposed, cap=existing)
|
||||
)
|
||||
|
||||
|
||||
def _looser_model_cap(proposed: BudgetConfig, cap: BudgetConfig) -> bool:
|
||||
return (
|
||||
proposed.max_budget is None
|
||||
or (cap.max_budget is not None and proposed.max_budget > cap.max_budget)
|
||||
or proposed.budget_duration != cap.budget_duration
|
||||
)
|
||||
|
||||
|
||||
def _should_auto_add_team_creator(
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import pytest
|
|||
from litellm.proxy.db.budget_window_spend_writer import (
|
||||
WindowSeedTotals,
|
||||
commit_window_spend_updates,
|
||||
open_window_spend_row,
|
||||
roll_window_spend_row,
|
||||
spend_logs_seed_totals,
|
||||
)
|
||||
|
|
@ -405,6 +406,29 @@ async def test_roll_window_spend_row_is_conditional_on_the_stored_window_being_o
|
|||
assert params[:4] == ("team", "t1", "30d", datetime(2026, 8, 31))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("replaces_window_start", "replaced_param"), [(None, None), (WINDOW_A, datetime(2026, 8, 1))]
|
||||
)
|
||||
async def test_open_window_spend_row_moves_only_the_window_the_caller_saw_end(replaces_window_start, replaced_param):
|
||||
db = _FakeDB()
|
||||
|
||||
await open_window_spend_row(
|
||||
prisma_client=_FakePrismaClient(db),
|
||||
entity_type="model_max_budget",
|
||||
entity_id="k1",
|
||||
window_duration="1d",
|
||||
window_start=WINDOW_B,
|
||||
replaces_window_start=replaces_window_start,
|
||||
)
|
||||
|
||||
((query, params),) = db.execute_raw_calls
|
||||
normalized = " ".join(query.split())
|
||||
assert "ON CONFLICT (entity_type, entity_id, window_duration) DO UPDATE" in normalized
|
||||
assert normalized.endswith("WHERE \"LiteLLM_BudgetWindowSpend\".window_start <= ($5::timestamptz AT TIME ZONE 'UTC')")
|
||||
assert params == ("model_max_budget", "k1", "1d", datetime(2026, 8, 31), replaced_param)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_seed_receives_the_batch_earliest_start_as_its_cutoff():
|
||||
db = _FakeDB(existing_rows=[])
|
||||
|
|
|
|||
193
tests/unit/proxy/hooks/test_model_budget_shared_windows.py
Normal file
193
tests/unit/proxy/hooks/test_model_budget_shared_windows.py
Normal file
|
|
@ -0,0 +1,193 @@
|
|||
from datetime import datetime, timedelta, timezone
|
||||
from types import SimpleNamespace
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.proxy.db.db_transaction_queue.window_spend_update_queue import WindowSpendTransaction
|
||||
from litellm.proxy.hooks.model_budget_shared_windows import DatabaseSharedWindows, SharedWindow
|
||||
|
||||
SPEND_KEY: Final = "virtual_key_spend:key-hash:claude-opus*:1d"
|
||||
WINDOW_START: Final = datetime(2026, 10, 20, 9, 30, tzinfo=timezone.utc)
|
||||
|
||||
|
||||
class _Table:
|
||||
def __init__(self, row: object | None = None, error: Exception | None = None) -> None:
|
||||
self.row = row
|
||||
self.error = error
|
||||
self.reads = 0
|
||||
|
||||
async def find_unique(self, where: object) -> object | None:
|
||||
self.reads = self.reads + 1
|
||||
if self.error is not None:
|
||||
raise self.error
|
||||
return self.row
|
||||
|
||||
|
||||
class _Database:
|
||||
def __init__(self, table: _Table) -> None:
|
||||
self.litellm_budgetwindowspend = table
|
||||
self.opened_with: tuple[object, ...] = ()
|
||||
|
||||
async def execute_raw(self, query: str, *params: object) -> int:
|
||||
self.opened_with = params
|
||||
self.litellm_budgetwindowspend.row = SimpleNamespace(window_start=params[3], spend=0.0)
|
||||
return 1
|
||||
|
||||
|
||||
class _OutracedDatabase:
|
||||
def __init__(self, table: _Table, winner: object) -> None:
|
||||
self.litellm_budgetwindowspend = table
|
||||
self._winner = winner
|
||||
|
||||
async def execute_raw(self, query: str, *params: object) -> int:
|
||||
self.litellm_budgetwindowspend.row = self._winner
|
||||
return 0
|
||||
|
||||
|
||||
class _Sink:
|
||||
def __init__(self) -> None:
|
||||
self.transactions: tuple[WindowSpendTransaction, ...] = ()
|
||||
|
||||
async def __call__(self, transaction: WindowSpendTransaction) -> None:
|
||||
self.transactions = (*self.transactions, transaction)
|
||||
|
||||
|
||||
class _LaggingReplica:
|
||||
def __init__(self, writer: _Database) -> None:
|
||||
self.litellm_budgetwindowspend = _Table()
|
||||
self._writer = writer
|
||||
|
||||
async def execute_raw(self, query: str, *params: object) -> int:
|
||||
return await self._writer.execute_raw(query, *params)
|
||||
|
||||
|
||||
def _database_windows(table: _Table | None, sink: _Sink, lagging_replica: bool = False) -> DatabaseSharedWindows:
|
||||
writer: Final = None if table is None else _Database(table)
|
||||
prisma: Final = (
|
||||
None
|
||||
if writer is None
|
||||
else SimpleNamespace(db=_LaggingReplica(writer) if lagging_replica else writer, writer_db=writer)
|
||||
)
|
||||
return DatabaseSharedWindows(
|
||||
prisma_client_source=lambda: prisma, # pyright: ignore[reportArgumentType] # duck-typed prisma double
|
||||
sink=sink,
|
||||
)
|
||||
|
||||
|
||||
def test_a_window_is_open_until_its_period_has_passed():
|
||||
window: Final = SharedWindow(started_at=WINDOW_START.timestamp(), spend=1.0)
|
||||
day: Final = 24 * 60 * 60
|
||||
|
||||
assert window.is_open(budget_duration="1d", now=WINDOW_START.timestamp() + day - 1)
|
||||
assert not window.is_open(budget_duration="1d", now=WINDOW_START.timestamp() + day)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_returns_the_stored_window_and_rereads_only_after_the_cache_expires():
|
||||
table: Final = _Table(row=SimpleNamespace(window_start=WINDOW_START.replace(tzinfo=None), spend=2.5))
|
||||
windows: Final = _database_windows(table, _Sink())
|
||||
|
||||
first: Final = await windows.read(spend_key=SPEND_KEY, budget_duration="1d")
|
||||
second: Final = await windows.read(spend_key=SPEND_KEY, budget_duration="1d")
|
||||
|
||||
assert first == SharedWindow(started_at=WINDOW_START.timestamp(), spend=2.5)
|
||||
assert second == first
|
||||
assert table.reads == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_unreadable_table_reads_as_no_shared_window():
|
||||
table: Final = _Table(error=RuntimeError("database is down"))
|
||||
windows: Final = _database_windows(table, _Sink())
|
||||
|
||||
assert await windows.read(spend_key=SPEND_KEY, budget_duration="1d") is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_queues_the_charge_for_the_window_it_belongs_to():
|
||||
sink: Final = _Sink()
|
||||
windows: Final = _database_windows(_Table(), sink)
|
||||
|
||||
await windows.add(spend_key=SPEND_KEY, budget_duration="1d", started_at=WINDOW_START.timestamp(), spend=0.25)
|
||||
|
||||
assert len(sink.transactions) == 1
|
||||
queued: Final = sink.transactions[0]
|
||||
assert (queued["entity_id"], queued["window_duration"], queued["spend"]) == (SPEND_KEY, "1d", 0.25)
|
||||
assert datetime.fromisoformat(queued["window_start"]).replace(tzinfo=timezone.utc) == WINDOW_START
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_nothing_is_queued_without_a_database():
|
||||
sink: Final = _Sink()
|
||||
windows: Final = _database_windows(None, sink)
|
||||
|
||||
await windows.add(spend_key=SPEND_KEY, budget_duration="1d", started_at=WINDOW_START.timestamp(), spend=0.25)
|
||||
|
||||
assert sink.transactions == ()
|
||||
assert await windows.read(spend_key=SPEND_KEY, budget_duration="1d") is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_open_joins_the_window_another_replica_already_started():
|
||||
table: Final = _Table(row=SimpleNamespace(window_start=WINDOW_START.replace(tzinfo=None), spend=2.5))
|
||||
windows: Final = _database_windows(table, _Sink())
|
||||
|
||||
started_at: Final = await windows.open(
|
||||
spend_key=SPEND_KEY, budget_duration="1d", now=WINDOW_START.timestamp() + 3600
|
||||
)
|
||||
|
||||
assert started_at == WINDOW_START.timestamp()
|
||||
assert table.row.spend == 2.5
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_open_reads_the_window_from_the_writer_while_the_read_replica_lags():
|
||||
table: Final = _Table(row=SimpleNamespace(window_start=WINDOW_START.replace(tzinfo=None), spend=2.5))
|
||||
windows: Final = _database_windows(table, _Sink(), lagging_replica=True)
|
||||
|
||||
started_at: Final = await windows.open(
|
||||
spend_key=SPEND_KEY, budget_duration="1d", now=WINDOW_START.timestamp() + 3600
|
||||
)
|
||||
|
||||
assert started_at == WINDOW_START.timestamp()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_open_joins_the_window_a_racing_replica_opened_first():
|
||||
winner: Final = SimpleNamespace(window_start=WINDOW_START.replace(tzinfo=None), spend=2.5)
|
||||
database: Final = _OutracedDatabase(_Table(), winner)
|
||||
windows: Final = DatabaseSharedWindows(
|
||||
prisma_client_source=lambda: SimpleNamespace(db=database, writer_db=database), # pyright: ignore[reportArgumentType] # duck-typed prisma double
|
||||
sink=_Sink(),
|
||||
)
|
||||
|
||||
started_at: Final = await windows.open(spend_key=SPEND_KEY, budget_duration="1d", now=WINDOW_START.timestamp() + 60)
|
||||
|
||||
assert started_at == WINDOW_START.timestamp()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("stored_start", [None, WINDOW_START - timedelta(days=2)])
|
||||
async def test_open_starts_a_window_at_now_when_none_is_open(stored_start: datetime | None):
|
||||
table: Final = _Table(
|
||||
row=None if stored_start is None else SimpleNamespace(window_start=stored_start.replace(tzinfo=None), spend=9.0)
|
||||
)
|
||||
windows: Final = _database_windows(table, _Sink())
|
||||
|
||||
started_at: Final = await windows.open(spend_key=SPEND_KEY, budget_duration="1d", now=WINDOW_START.timestamp())
|
||||
|
||||
assert started_at == WINDOW_START.timestamp()
|
||||
assert await windows.read(spend_key=SPEND_KEY, budget_duration="1d") == SharedWindow(
|
||||
started_at=WINDOW_START.timestamp(), spend=0.0
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("table", [None, _Table(error=RuntimeError("database is down"))])
|
||||
async def test_open_falls_back_to_this_replicas_own_window_without_a_database(table: _Table | None):
|
||||
windows: Final = _database_windows(table, _Sink())
|
||||
|
||||
assert await windows.open(spend_key=SPEND_KEY, budget_duration="1d", now=WINDOW_START.timestamp()) == (
|
||||
WINDOW_START.timestamp()
|
||||
)
|
||||
|
|
@ -1,6 +1,9 @@
|
|||
import asyncio
|
||||
import functools
|
||||
import re
|
||||
import time
|
||||
from collections.abc import Callable, Mapping
|
||||
from datetime import datetime, timezone
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
|
|
@ -8,12 +11,19 @@ import pytest
|
|||
|
||||
import litellm
|
||||
from litellm.caching.caching import DualCache
|
||||
from litellm.proxy._types import Litellm_EntityType
|
||||
from litellm.proxy._types import Litellm_EntityType, UserAPIKeyAuth
|
||||
from litellm.proxy.hooks.model_budget_shared_windows import SharedWindow, SharedWindows
|
||||
from litellm.proxy.hooks.model_max_budget_limiter import (
|
||||
PROXY_VirtualKeyModelMaxBudgetLimiter,
|
||||
build_model_max_budget_usage,
|
||||
model_access_groups_of,
|
||||
model_budget_spend_cache_key,
|
||||
resolve_model_budget,
|
||||
routed_models_of,
|
||||
team_member_budget_entity_id,
|
||||
team_model_budget_applies,
|
||||
)
|
||||
from litellm.router import Router
|
||||
from litellm.types.caching import RedisPipelineIncrementOperation
|
||||
from litellm.types.utils import LiteLLMBatch, Usage
|
||||
|
||||
|
|
@ -74,10 +84,14 @@ def _event(
|
|||
}
|
||||
|
||||
|
||||
async def _log_success(
|
||||
limiter: PROXY_VirtualKeyModelMaxBudgetLimiter, event: dict[str, object], response_obj: object = None
|
||||
) -> None:
|
||||
await limiter.async_log_success_event(event, response_obj=response_obj, start_time=None, end_time=None)
|
||||
|
||||
|
||||
async def _poll(limiter: PROXY_VirtualKeyModelMaxBudgetLimiter, batch: LiteLLMBatch, response_cost: float) -> None:
|
||||
await limiter.async_log_success_event(
|
||||
_event("aretrieve_batch", response_cost), response_obj=batch, start_time=None, end_time=None
|
||||
)
|
||||
await _log_success(limiter, _event("aretrieve_batch", response_cost), response_obj=batch)
|
||||
|
||||
|
||||
async def _chat(
|
||||
|
|
@ -86,7 +100,8 @@ async def _chat(
|
|||
user_id: str = USER_ID,
|
||||
team_id: str = TEAM_ID,
|
||||
) -> None:
|
||||
await limiter.async_log_success_event(
|
||||
await _log_success(
|
||||
limiter,
|
||||
_event(
|
||||
"acompletion",
|
||||
CHAT_COST,
|
||||
|
|
@ -94,9 +109,6 @@ async def _chat(
|
|||
user_id=user_id,
|
||||
team_id=team_id,
|
||||
),
|
||||
response_obj=None,
|
||||
start_time=None,
|
||||
end_time=None,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -311,3 +323,399 @@ async def test_a_batch_polled_within_every_budget_window_is_never_charged_again(
|
|||
await _poll(limiter, finished, BATCH_COST)
|
||||
|
||||
assert _local_spend(limiter, KEY_SPEND_KEY) == pytest.approx(BATCH_COST)
|
||||
|
||||
|
||||
OPUS_GROUP: Final = "claude-opus"
|
||||
OPUS_GROUP_MEMBERS: Final = MappingProxyType(
|
||||
{"claude-opus-4-8": frozenset({OPUS_GROUP}), "claude-opus-5-5": frozenset({OPUS_GROUP})}
|
||||
)
|
||||
FAMILY_CAP: Final = {"budget_limit": CHAT_COST, "time_period": "1d"} # mutable-ok: BudgetConfig validates dicts only
|
||||
ROOMY_CAP: Final = {
|
||||
"budget_limit": CHAT_COST * 100,
|
||||
"time_period": "1d",
|
||||
} # mutable-ok: BudgetConfig validates dicts only
|
||||
WINDOW_EPOCH: Final = datetime(2026, 10, 20, 9, 30, tzinfo=timezone.utc).timestamp()
|
||||
|
||||
|
||||
def _model_access_groups(groups_by_model: Mapping[str, frozenset[str]]) -> Callable[[str], frozenset[str]]:
|
||||
return lambda model: groups_by_model.get(model, frozenset())
|
||||
|
||||
|
||||
class _SharedWindowsDouble:
|
||||
def __init__(self) -> None:
|
||||
self.rows: Mapping[str, SharedWindow] = MappingProxyType({})
|
||||
self.queued: tuple[tuple[str, float, float], ...] = ()
|
||||
|
||||
async def read(self, spend_key: str, budget_duration: str) -> SharedWindow | None:
|
||||
return self.rows.get(spend_key)
|
||||
|
||||
async def open(self, spend_key: str, budget_duration: str, now: float) -> float:
|
||||
await asyncio.sleep(0)
|
||||
stored: Final = self.rows.get(spend_key)
|
||||
if stored is not None and stored.is_open(budget_duration=budget_duration, now=now):
|
||||
return stored.started_at
|
||||
self.rows = MappingProxyType({**self.rows, spend_key: SharedWindow(started_at=now, spend=0.0)})
|
||||
return now
|
||||
|
||||
async def add(self, spend_key: str, budget_duration: str, started_at: float, spend: float) -> None:
|
||||
self.queued = (*self.queued, (spend_key, started_at, spend))
|
||||
|
||||
def flush(self) -> None:
|
||||
self.rows = functools.reduce(_upsert_window_spend, self.queued, self.rows)
|
||||
self.queued = ()
|
||||
|
||||
|
||||
def _upsert_window_spend(
|
||||
rows: Mapping[str, SharedWindow], queued: tuple[str, float, float]
|
||||
) -> Mapping[str, SharedWindow]:
|
||||
spend_key, started_at, spend = queued
|
||||
stored: Final = rows.get(spend_key)
|
||||
row: Final = (
|
||||
SharedWindow(started_at=stored.started_at, spend=stored.spend + spend)
|
||||
if stored is not None and stored.started_at >= started_at
|
||||
else SharedWindow(started_at=started_at, spend=spend)
|
||||
)
|
||||
return MappingProxyType({**rows, spend_key: row})
|
||||
|
||||
|
||||
def _family_limiter(
|
||||
groups_by_model: Mapping[str, frozenset[str]] = OPUS_GROUP_MEMBERS,
|
||||
dual_cache: DualCache | None = None,
|
||||
shared_windows: SharedWindows | None = None,
|
||||
clock: Callable[[], float] = time.time,
|
||||
) -> PROXY_VirtualKeyModelMaxBudgetLimiter:
|
||||
return PROXY_VirtualKeyModelMaxBudgetLimiter(
|
||||
dual_cache=dual_cache or DualCache(),
|
||||
clock=clock,
|
||||
model_access_groups=_model_access_groups(groups_by_model),
|
||||
shared_windows=shared_windows or _SharedWindowsDouble(),
|
||||
)
|
||||
|
||||
|
||||
def _key(model_max_budget: Mapping[str, object]) -> UserAPIKeyAuth:
|
||||
return UserAPIKeyAuth(token=KEY_HASH, key_alias="opus-family", model_max_budget=dict(model_max_budget))
|
||||
|
||||
|
||||
async def _spend_on(
|
||||
limiter: PROXY_VirtualKeyModelMaxBudgetLimiter, model: str, model_max_budget: Mapping[str, object]
|
||||
) -> None:
|
||||
await _log_success(
|
||||
limiter,
|
||||
{
|
||||
"call_type": "acompletion",
|
||||
"standard_logging_object": {
|
||||
"call_type": "acompletion",
|
||||
"response_cost": CHAT_COST,
|
||||
"model": f"anthropic/{model}",
|
||||
"model_group": model,
|
||||
"metadata": {"user_api_key_hash": KEY_HASH},
|
||||
},
|
||||
"litellm_params": {"metadata": {"user_api_key_model_max_budget": dict(model_max_budget)}},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("family_entry", ["claude-opus*", OPUS_GROUP])
|
||||
async def test_every_opus_version_draws_from_one_family_entry(family_entry: str):
|
||||
budget: Final = {family_entry: FAMILY_CAP}
|
||||
limiter: Final = _family_limiter()
|
||||
assert await limiter.is_key_within_model_budget(user_api_key_dict=_key(budget), model="claude-opus-5-5")
|
||||
|
||||
await _spend_on(limiter, model="claude-opus-4-8", model_max_budget=budget)
|
||||
|
||||
with pytest.raises(
|
||||
litellm.BudgetExceededError, match=rf"model=claude-opus-5-5, budget_model={re.escape(family_entry)}$"
|
||||
):
|
||||
await limiter.is_key_within_model_budget(user_api_key_dict=_key(budget), model="claude-opus-5-5")
|
||||
assert await limiter.is_key_within_model_budget(user_api_key_dict=_key(budget), model="claude-sonnet-5")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_model_added_to_the_access_group_later_is_covered_with_no_key_change():
|
||||
budget: Final = {OPUS_GROUP: FAMILY_CAP}
|
||||
shared_cache: Final = DualCache()
|
||||
before_group_change: Final = _family_limiter(dual_cache=shared_cache)
|
||||
after_group_change: Final = _family_limiter(
|
||||
groups_by_model={**OPUS_GROUP_MEMBERS, "claude-opus-6": frozenset({OPUS_GROUP})}, dual_cache=shared_cache
|
||||
)
|
||||
|
||||
await _spend_on(before_group_change, model="claude-opus-4-8", model_max_budget=budget)
|
||||
|
||||
assert await before_group_change.is_key_within_model_budget(user_api_key_dict=_key(budget), model="claude-opus-6")
|
||||
with pytest.raises(litellm.BudgetExceededError):
|
||||
await after_group_change.is_key_within_model_budget(user_api_key_dict=_key(budget), model="claude-opus-6")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("budget", "checked_model", "blocked"),
|
||||
[
|
||||
({"claude-opus-4-8": ROOMY_CAP, "claude-opus*": FAMILY_CAP}, "claude-opus-4-8", False),
|
||||
({"claude-opus-4-8": ROOMY_CAP, "claude-opus*": FAMILY_CAP}, "claude-opus-5-5", False),
|
||||
({OPUS_GROUP: FAMILY_CAP, "claude-opus*": ROOMY_CAP}, "claude-opus-5-5", True),
|
||||
({"claude-opus-4*": FAMILY_CAP, "claude-opus*": ROOMY_CAP}, "claude-opus-4-7", True),
|
||||
({"claude-opus-4*": FAMILY_CAP, "claude-opus*": ROOMY_CAP}, "claude-opus-5-5", False),
|
||||
({"claude-opus-4*": {"budget_limit": CHAT_COST}, "claude-opus*": FAMILY_CAP}, "claude-opus-5-5", True),
|
||||
({"claude-opus-4*": FAMILY_CAP, "c" + "*" * 15: ROOMY_CAP}, "claude-opus-4-7", True),
|
||||
],
|
||||
)
|
||||
async def test_the_most_specific_entry_takes_the_spend_and_gates_the_request(
|
||||
budget: Mapping[str, object], checked_model: str, blocked: bool
|
||||
):
|
||||
limiter: Final = _family_limiter()
|
||||
|
||||
await _spend_on(limiter, model="claude-opus-4-8", model_max_budget=budget)
|
||||
|
||||
if blocked:
|
||||
with pytest.raises(litellm.BudgetExceededError):
|
||||
await limiter.is_key_within_model_budget(user_api_key_dict=_key(budget), model=checked_model)
|
||||
else:
|
||||
assert await limiter.is_key_within_model_budget(user_api_key_dict=_key(budget), model=checked_model)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_exact_entry_refusal_reads_as_before_for_a_provider_prefixed_model():
|
||||
budget: Final = {"claude-opus-4-8": FAMILY_CAP}
|
||||
limiter: Final = _family_limiter()
|
||||
|
||||
await _spend_on(limiter, model="claude-opus-4-8", model_max_budget=budget)
|
||||
|
||||
with pytest.raises(litellm.BudgetExceededError, match=r"model=anthropic/claude-opus-4-8$"):
|
||||
await limiter.is_key_within_model_budget(user_api_key_dict=_key(budget), model="anthropic/claude-opus-4-8")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("key_entry", "team_cap_applies"),
|
||||
[("claude-opus-4-8", False), ("anthropic/claude-opus-4-8", True), ("claude-opus*", True), (OPUS_GROUP, True)],
|
||||
)
|
||||
def test_only_a_key_entry_named_after_the_model_replaces_the_team_cap(key_entry: str, team_cap_applies: bool):
|
||||
assert team_model_budget_applies(model="claude-opus-4-8", key_model_max_budget={key_entry: ROOMY_CAP}) is (
|
||||
team_cap_applies
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_end_user_windows_stay_on_the_replica_without_redis():
|
||||
budget: Final = {"claude-opus*": FAMILY_CAP}
|
||||
table: Final = _SharedWindowsDouble()
|
||||
limiter: Final = _family_limiter(shared_windows=table)
|
||||
|
||||
await _log_success(
|
||||
limiter,
|
||||
{
|
||||
"call_type": "acompletion",
|
||||
"standard_logging_object": {
|
||||
"call_type": "acompletion",
|
||||
"response_cost": CHAT_COST,
|
||||
"model": "anthropic/claude-opus-4-8",
|
||||
"model_group": "claude-opus-4-8",
|
||||
"end_user": "end-user-1",
|
||||
"metadata": {},
|
||||
},
|
||||
"litellm_params": {"metadata": {"user_api_key_end_user_model_max_budget": budget}},
|
||||
},
|
||||
)
|
||||
table.flush()
|
||||
|
||||
assert dict(table.rows) == {}
|
||||
with pytest.raises(litellm.BudgetExceededError, match="model=claude-opus-5-5"):
|
||||
await limiter.is_end_user_within_model_budget(
|
||||
end_user_id="end-user-1", end_user_model_max_budget=budget, model="claude-opus-5-5"
|
||||
)
|
||||
|
||||
|
||||
def test_an_access_group_named_after_a_served_model_never_widens_that_models_entry():
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "gpt-4o",
|
||||
"litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-test"},
|
||||
"model_info": {"access_groups": ["gpt-4o", "openai-models"]},
|
||||
},
|
||||
{
|
||||
"model_name": "gpt-4o-mini",
|
||||
"litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-test"},
|
||||
"model_info": {"access_groups": ["gpt-4o", "openai-models"]},
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
def governing_entry(model_max_budget: Mapping[str, object]) -> str | None:
|
||||
resolved: Final = resolve_model_budget(
|
||||
model="gpt-4o-mini",
|
||||
model_max_budget=model_max_budget,
|
||||
model_access_groups=lambda model: model_access_groups_of(router=router, model=model),
|
||||
)
|
||||
return None if resolved is None else resolved.budget_model
|
||||
|
||||
assert governing_entry({"gpt-4o": FAMILY_CAP}) is None
|
||||
assert governing_entry({"openai-models": FAMILY_CAP}) == "openai-models"
|
||||
assert routed_models_of(router).members("gpt-4o") == frozenset()
|
||||
assert routed_models_of(router).members("openai-models") == frozenset({"gpt-4o", "gpt-4o-mini"})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_key_info_usage_reports_the_family_total():
|
||||
budget: Final = {"claude-opus*": ROOMY_CAP}
|
||||
limiter: Final = _family_limiter()
|
||||
|
||||
await _spend_on(limiter, model="claude-opus-4-8", model_max_budget=budget)
|
||||
await _spend_on(limiter, model="claude-opus-5-5", model_max_budget=budget)
|
||||
|
||||
usage: Final = await build_model_max_budget_usage(
|
||||
entity_type=Litellm_EntityType.KEY,
|
||||
entity_id=KEY_HASH,
|
||||
model_max_budget=budget,
|
||||
cache=limiter.dual_cache,
|
||||
)
|
||||
assert usage["claude-opus*"]["current_spend"] == round(CHAT_COST * 2, 4)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("pattern", "model", "governs"),
|
||||
[
|
||||
("claude-opus*", "claude-opus-4-8", True),
|
||||
("claude-opus*", "anthropic/claude-opus-4-8", True),
|
||||
("anthropic/claude-opus*", "anthropic/claude-opus-4-8", True),
|
||||
("anthropic/claude-opus*", "claude-opus-4-8", False),
|
||||
("claude-opus*", "my-claude-opus-4-8", False),
|
||||
("claude-opus*", "claude-sonnet-5", False),
|
||||
("gpt-4.1*", "gpt-401-mini", False),
|
||||
("*", "claude-sonnet-5", True),
|
||||
],
|
||||
)
|
||||
def test_glob_entries_match_the_whole_model_name_with_only_star_as_a_wildcard(pattern: str, model: str, governs: bool):
|
||||
resolved: Final = resolve_model_budget(
|
||||
model=model, model_max_budget={pattern: FAMILY_CAP}, model_access_groups=_model_access_groups({})
|
||||
)
|
||||
|
||||
assert (resolved is not None and resolved.budget_model == pattern) is governs
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_replicas_without_redis_enforce_one_family_cap_together():
|
||||
budget: Final = {"claude-opus*": FAMILY_CAP}
|
||||
table: Final = _SharedWindowsDouble()
|
||||
replica_a: Final = _family_limiter(shared_windows=table)
|
||||
replica_b: Final = _family_limiter(shared_windows=table)
|
||||
|
||||
await _spend_on(replica_a, model="claude-opus-4-8", model_max_budget=budget)
|
||||
table.flush()
|
||||
|
||||
with pytest.raises(litellm.BudgetExceededError, match="model=claude-opus-5-5"):
|
||||
await replica_b.is_key_within_model_budget(user_api_key_dict=_key(budget), model="claude-opus-5-5")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_replicas_sharing_redis_enforce_the_family_cap_without_the_database():
|
||||
budget: Final = {"claude-opus*": FAMILY_CAP}
|
||||
redis: Final = _SharedRedisDouble()
|
||||
table: Final = _SharedWindowsDouble()
|
||||
replica_a: Final = _family_limiter(
|
||||
dual_cache=DualCache(redis_cache=redis), # pyright: ignore[reportArgumentType] # duck-typed Redis double
|
||||
shared_windows=table,
|
||||
)
|
||||
replica_b: Final = _family_limiter(
|
||||
dual_cache=DualCache(redis_cache=redis), # pyright: ignore[reportArgumentType] # duck-typed Redis double
|
||||
shared_windows=table,
|
||||
)
|
||||
|
||||
await _spend_on(replica_a, model="claude-opus-4-8", model_max_budget=budget)
|
||||
await _drain_redis_pushes()
|
||||
table.flush()
|
||||
|
||||
with pytest.raises(litellm.BudgetExceededError, match="model=claude-opus-5-5"):
|
||||
await replica_b.is_key_within_model_budget(user_api_key_dict=_key(budget), model="claude-opus-5-5")
|
||||
assert dict(table.rows) == {}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_replica_charges_the_open_shared_window_instead_of_restarting_it():
|
||||
budget: Final = {"claude-opus*": {"budget_limit": CHAT_COST * 1.5, "time_period": "1d"}}
|
||||
table: Final = _SharedWindowsDouble()
|
||||
clock: Final = _Clock()
|
||||
clock.advance(WINDOW_EPOCH)
|
||||
replica_a: Final = _family_limiter(shared_windows=table, clock=clock.now)
|
||||
replica_b: Final = _family_limiter(shared_windows=table, clock=clock.now)
|
||||
|
||||
await _spend_on(replica_a, model="claude-opus-4-8", model_max_budget=budget)
|
||||
table.flush()
|
||||
clock.advance(3600)
|
||||
await _spend_on(replica_b, model="claude-opus-5-5", model_max_budget=budget)
|
||||
table.flush()
|
||||
|
||||
for replica in (replica_a, replica_b):
|
||||
with pytest.raises(litellm.BudgetExceededError):
|
||||
await replica.is_key_within_model_budget(user_api_key_dict=_key(budget), model="claude-opus-5-5")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_key_info_on_one_replica_reports_spend_from_every_replica():
|
||||
budget: Final = {"claude-opus*": ROOMY_CAP}
|
||||
table: Final = _SharedWindowsDouble()
|
||||
replica_a: Final = _family_limiter(shared_windows=table)
|
||||
replica_b: Final = _family_limiter(shared_windows=table)
|
||||
|
||||
await _spend_on(replica_a, model="claude-opus-4-8", model_max_budget=budget)
|
||||
await _spend_on(replica_b, model="claude-opus-5-5", model_max_budget=budget)
|
||||
table.flush()
|
||||
|
||||
usage: Final = await build_model_max_budget_usage(
|
||||
entity_type=Litellm_EntityType.KEY,
|
||||
entity_id=KEY_HASH,
|
||||
model_max_budget=budget,
|
||||
cache=replica_a.dual_cache,
|
||||
shared_windows=table,
|
||||
)
|
||||
assert usage["claude-opus*"]["current_spend"] == round(CHAT_COST * 2, 4)
|
||||
assert usage["claude-opus*"]["budget_reset_at"] is not None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("budget_model", ["claude-opus*", "claude-opus-4-8"])
|
||||
async def test_replicas_starting_a_window_moments_apart_keep_each_others_spend(budget_model: str):
|
||||
budget: Final = {budget_model: ROOMY_CAP}
|
||||
table: Final = _SharedWindowsDouble()
|
||||
clock: Final = _Clock()
|
||||
clock.advance(WINDOW_EPOCH)
|
||||
replica_a: Final = _family_limiter(shared_windows=table, clock=clock.now)
|
||||
replica_b: Final = _family_limiter(shared_windows=table, clock=clock.now)
|
||||
|
||||
await _spend_on(replica_a, model="claude-opus-4-8", model_max_budget=budget)
|
||||
clock.advance(1)
|
||||
await _spend_on(replica_b, model="claude-opus-4-8", model_max_budget=budget)
|
||||
table.flush()
|
||||
|
||||
usage: Final = await build_model_max_budget_usage(
|
||||
entity_type=Litellm_EntityType.KEY,
|
||||
entity_id=KEY_HASH,
|
||||
model_max_budget=budget,
|
||||
cache=DualCache(),
|
||||
now=datetime.fromtimestamp(clock.now(), tz=timezone.utc),
|
||||
shared_windows=table,
|
||||
)
|
||||
assert usage[budget_model]["current_spend"] == round(CHAT_COST * 2, 4)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_concurrent_first_charges_on_one_replica_both_count():
|
||||
budget: Final = {"claude-opus*": ROOMY_CAP}
|
||||
clock: Final = _Clock()
|
||||
clock.advance(WINDOW_EPOCH)
|
||||
limiter: Final = _family_limiter(shared_windows=_SharedWindowsDouble(), clock=clock.now)
|
||||
|
||||
await asyncio.gather(
|
||||
_spend_on(limiter, model="claude-opus-4-8", model_max_budget=budget),
|
||||
_spend_on(limiter, model="claude-opus-5-5", model_max_budget=budget),
|
||||
)
|
||||
|
||||
usage: Final = await build_model_max_budget_usage(
|
||||
entity_type=Litellm_EntityType.KEY,
|
||||
entity_id=KEY_HASH,
|
||||
model_max_budget=budget,
|
||||
cache=limiter.dual_cache,
|
||||
now=datetime.fromtimestamp(clock.now(), tz=timezone.utc),
|
||||
shared_windows=_SharedWindowsDouble(),
|
||||
)
|
||||
assert usage["claude-opus*"]["current_spend"] == round(CHAT_COST * 2, 4)
|
||||
|
|
|
|||
|
|
@ -16580,6 +16580,7 @@ async def test_info_key_fn_includes_model_max_budget_usage(monkeypatch):
|
|||
}
|
||||
|
||||
mock_prisma_client = AsyncMock()
|
||||
mock_prisma_client.db.litellm_budgetwindowspend.find_unique = AsyncMock(return_value=None)
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client)
|
||||
mock_user_api_key_cache = AsyncMock()
|
||||
monkeypatch.setattr(
|
||||
|
|
@ -16703,6 +16704,7 @@ async def test_info_key_fn_v2_includes_model_max_budget_usage(monkeypatch):
|
|||
model_max_budget = {"gpt-4o": {"budget_limit": 1.00, "time_period": "7d"}}
|
||||
|
||||
mock_prisma_client = AsyncMock()
|
||||
mock_prisma_client.db.litellm_budgetwindowspend.find_unique = AsyncMock(return_value=None)
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client)
|
||||
mock_user_api_key_cache = AsyncMock()
|
||||
monkeypatch.setattr(
|
||||
|
|
@ -16767,6 +16769,7 @@ async def test_info_key_fn_budget_table_fallback(monkeypatch):
|
|||
}
|
||||
|
||||
mock_prisma_client = AsyncMock()
|
||||
mock_prisma_client.db.litellm_budgetwindowspend.find_unique = AsyncMock(return_value=None)
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client)
|
||||
mock_user_api_key_cache = AsyncMock()
|
||||
monkeypatch.setattr(
|
||||
|
|
@ -16840,6 +16843,7 @@ async def test_info_key_fn_v2_budget_table_fallback(monkeypatch):
|
|||
}
|
||||
|
||||
mock_prisma_client = AsyncMock()
|
||||
mock_prisma_client.db.litellm_budgetwindowspend.find_unique = AsyncMock(return_value=None)
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client)
|
||||
mock_user_api_key_cache = AsyncMock()
|
||||
monkeypatch.setattr(
|
||||
|
|
@ -17215,6 +17219,7 @@ async def test_info_key_fn_reads_the_configured_budget_model_key(monkeypatch):
|
|||
}
|
||||
|
||||
mock_prisma_client = AsyncMock()
|
||||
mock_prisma_client.db.litellm_budgetwindowspend.find_unique = AsyncMock(return_value=None)
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client)
|
||||
mock_user_api_key_cache = AsyncMock()
|
||||
monkeypatch.setattr(
|
||||
|
|
|
|||
|
|
@ -44,6 +44,7 @@ from litellm.proxy._types import (
|
|||
UpdateTeamRequest,
|
||||
UserAPIKeyAuth, # Import UserAPIKeyAuth
|
||||
)
|
||||
from litellm.proxy.hooks.model_max_budget_limiter import RoutedModels
|
||||
from litellm.proxy.management.teams.authz import TeamAccess
|
||||
from litellm.proxy.management_endpoints.team_admin_field_permissions import (
|
||||
PROXY_CONTROLLED_TEAM_METADATA_KEYS,
|
||||
|
|
@ -17593,6 +17594,130 @@ def test_team_admin_can_tighten_or_keep_team_model_caps(requested) -> None:
|
|||
)
|
||||
|
||||
|
||||
_TEAM_OPUS_FAMILY_CAP: Final = MappingProxyType({"claude-opus*": {"max_budget": 10.0, "budget_duration": "1d"}})
|
||||
_TEAM_OPUS_GROUP_CAP: Final = MappingProxyType({"z-opus-group": {"max_budget": 10.0, "budget_duration": "1d"}})
|
||||
_TEAM_ROUTED_MODELS: Final = RoutedModels(
|
||||
served_names=frozenset(
|
||||
{"claude-opus-4-8", "claude-opus-5-5", "gemini-3-pro", "gpt-4o", "my-team/gpt-4o", "team-route/*"}
|
||||
),
|
||||
group_members=MappingProxyType(
|
||||
{
|
||||
"opus-group": frozenset({"claude-opus-4-8"}),
|
||||
"z-opus-group": frozenset({"claude-opus-4-8"}),
|
||||
"a-opus-group": frozenset({"claude-opus-4-8", "claude-opus-5-5"}),
|
||||
"zz-opus-group": frozenset({"claude-opus-4-8"}),
|
||||
"a-gemini-group": frozenset({"gemini-3-pro"}),
|
||||
"z-anthropic-group": frozenset({"anthropic/*"}),
|
||||
}
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _check_team_admin_model_caps(existing: Mapping[str, object], requested: Mapping[str, object]) -> None:
|
||||
from litellm.proxy.management_endpoints.team_endpoints import _check_team_model_budget_update_authority
|
||||
|
||||
_check_team_model_budget_update_authority(
|
||||
data=UpdateTeamRequest(team_id="t1", model_max_budget=dict(requested)),
|
||||
user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, user_id="team-admin"),
|
||||
existing_model_max_budget=existing,
|
||||
routed_models_source=lambda: _TEAM_ROUTED_MODELS,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("existing", "added"),
|
||||
[
|
||||
(_TEAM_OPUS_FAMILY_CAP, {"opus-group": {"max_budget": 1000.0, "budget_duration": "1d"}}),
|
||||
(_TEAM_OPUS_FAMILY_CAP, {"opus-group": {"max_budget": 10.0, "budget_duration": "7d"}}),
|
||||
(_TEAM_OPUS_FAMILY_CAP, {"*laude-opus-4-8": {"max_budget": 1000.0, "budget_duration": "1d"}}),
|
||||
(_TEAM_OPUS_GROUP_CAP, {"a-opus-group": {"max_budget": 1000.0, "budget_duration": "1d"}}),
|
||||
(_TEAM_OPUS_GROUP_CAP, {"claude-opus-4-8": {"max_budget": 1000.0, "budget_duration": "1d"}}),
|
||||
(
|
||||
{"z-anthropic-group": {"max_budget": 10.0, "budget_duration": "1d"}},
|
||||
{"claude-opus-4-8": {"max_budget": 1000.0, "budget_duration": "1d"}},
|
||||
),
|
||||
(_TEAM_OPUS_FAMILY_CAP, {"claude-opus-4-8": {"max_budget": 1000.0, "budget_duration": "1d"}}),
|
||||
(
|
||||
{"anthropic/*": {"max_budget": 10.0, "budget_duration": "1d"}},
|
||||
{"claude-opus-4-8": {"max_budget": 1000.0, "budget_duration": "1d"}},
|
||||
),
|
||||
(
|
||||
{"my-team/*": {"max_budget": 10.0, "budget_duration": "1d"}},
|
||||
{"gpt-4o": {"max_budget": 1000.0, "budget_duration": "1d"}},
|
||||
),
|
||||
(
|
||||
{"team-route/*": {"max_budget": 10.0, "budget_duration": "1d"}},
|
||||
{"gpt-4o": {"max_budget": 1000.0, "budget_duration": "1d"}},
|
||||
),
|
||||
],
|
||||
ids=[
|
||||
"group_over_pattern",
|
||||
"group_rewindows_pattern",
|
||||
"more_specific_pattern",
|
||||
"earlier_group_sharing_a_model",
|
||||
"model_inside_group",
|
||||
"model_under_wildcard_group",
|
||||
"model_matching_pattern",
|
||||
"model_under_provider_pattern",
|
||||
"model_under_served_prefix",
|
||||
"model_under_wildcard_route",
|
||||
],
|
||||
)
|
||||
def test_team_admin_cannot_add_an_entry_looser_than_a_cap_it_outranks(
|
||||
existing: Mapping[str, object], added: Mapping[str, object]
|
||||
) -> None:
|
||||
with pytest.raises(HTTPException) as exc:
|
||||
_check_team_admin_model_caps(existing=existing, requested={**existing, **added})
|
||||
assert exc.value.status_code == 403
|
||||
assert "proxy admin" in exc.value.detail["error"]
|
||||
assert repr(next(iter(existing))) in exc.value.detail["error"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("existing", "requested"),
|
||||
[
|
||||
(_TEAM_OPUS_FAMILY_CAP, {**_TEAM_OPUS_FAMILY_CAP, "opus-group": {"max_budget": 5.0, "budget_duration": "1d"}}),
|
||||
(_TEAM_OPUS_FAMILY_CAP, {**_TEAM_OPUS_FAMILY_CAP, "claude*": {"max_budget": 1000.0, "budget_duration": "1d"}}),
|
||||
(_TEAM_OPUS_GROUP_CAP, {**_TEAM_OPUS_GROUP_CAP, "a-gemini-group": {"max_budget": 1000.0, "budget_duration": "1d"}}),
|
||||
(_TEAM_OPUS_GROUP_CAP, {**_TEAM_OPUS_GROUP_CAP, "zz-opus-group": {"max_budget": 1000.0, "budget_duration": "1d"}}),
|
||||
(_TEAM_OPUS_GROUP_CAP, {**_TEAM_OPUS_GROUP_CAP, "gpt-4o": {"max_budget": 1000.0, "budget_duration": "1d"}}),
|
||||
(_TEAM_OPUS_FAMILY_CAP, {**_TEAM_OPUS_FAMILY_CAP, "gpt-4o": {"max_budget": 1000.0, "budget_duration": "1d"}}),
|
||||
(
|
||||
_TEAM_OPUS_FAMILY_CAP,
|
||||
{**_TEAM_OPUS_FAMILY_CAP, "a-gemini-group": {"max_budget": 1000.0, "budget_duration": "1d"}},
|
||||
),
|
||||
(
|
||||
{**_TEAM_OPUS_FAMILY_CAP, "opus-group": {"max_budget": 100.0, "budget_duration": "1d"}},
|
||||
{**_TEAM_OPUS_FAMILY_CAP, "opus-group": {"max_budget": 100.0, "budget_duration": "1d"}},
|
||||
),
|
||||
(
|
||||
{**_TEAM_OPUS_FAMILY_CAP, "opus-group": {"max_budget": 100.0, "budget_duration": "1d"}},
|
||||
{**_TEAM_OPUS_FAMILY_CAP, "opus-group": {"max_budget": 50.0, "budget_duration": "1d"}},
|
||||
),
|
||||
(
|
||||
{**_TEAM_OPUS_FAMILY_CAP, "opus-group": {"rpm_limit": 10}},
|
||||
{**_TEAM_OPUS_FAMILY_CAP, "opus-group": {"rpm_limit": 20}},
|
||||
),
|
||||
],
|
||||
ids=[
|
||||
"tighter_group",
|
||||
"less_specific_pattern",
|
||||
"group_without_shared_models",
|
||||
"later_group_sharing_a_model",
|
||||
"model_outside_group",
|
||||
"model_outside_pattern",
|
||||
"group_outside_pattern",
|
||||
"unchanged_looser_entry",
|
||||
"tightened_looser_entry",
|
||||
"uncapped_entry_edited",
|
||||
],
|
||||
)
|
||||
def test_team_admin_can_add_entries_that_never_lift_an_outranked_cap(
|
||||
existing: Mapping[str, object], requested: Mapping[str, object]
|
||||
) -> None:
|
||||
assert _check_team_admin_model_caps(existing=existing, requested=requested) is None
|
||||
|
||||
|
||||
def test_team_model_cap_authority_skips_omitted_field_malformed_rows_and_proxy_admins() -> None:
|
||||
from litellm.proxy.management_endpoints.team_endpoints import _check_team_model_budget_update_authority
|
||||
|
||||
|
|
|
|||
|
|
@ -74,6 +74,7 @@ _IN_MEMORY_ONLY_CALLERS: Final = frozenset(
|
|||
"litellm/proxy/container_endpoints/ownership.py",
|
||||
"litellm/proxy/discovery_endpoints/agent_skills_endpoints.py",
|
||||
"litellm/proxy/guardrails/guardrail_hooks/straiker/straiker.py",
|
||||
"litellm/proxy/hooks/model_budget_shared_windows.py",
|
||||
"litellm/proxy/spend_tracking/key_metadata_recovery.py",
|
||||
"litellm/proxy/ui_crud_endpoints/latest_release_endpoints.py",
|
||||
"litellm/responses/litellm_completion_transformation/transformation.py",
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import userEvent from "@testing-library/user-event";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { renderWithProviders, screen } from "../../../tests/test-utils";
|
||||
import { fireEvent, renderWithProviders, screen } from "../../../tests/test-utils";
|
||||
import { MODEL_MAX_BUDGET_PREMIUM_HINT, ModelMaxBudgetEditor, type ModelMaxBudget } from "./ModelMaxBudgetEditor";
|
||||
|
||||
const STORED: ModelMaxBudget = { "gpt-4o": { budget_limit: 5, time_period: "30d" } };
|
||||
|
|
@ -67,3 +68,65 @@ describe("ModelMaxBudgetEditor with an enterprise license", () => {
|
|||
expect(addButton()).toBeEnabled();
|
||||
});
|
||||
});
|
||||
|
||||
describe("ModelMaxBudgetEditor model families", () => {
|
||||
it("stores a typed pattern that matches no listed model as one budget entry", async () => {
|
||||
const user = userEvent.setup();
|
||||
const onChange = vi.fn();
|
||||
renderWithProviders(
|
||||
<ModelMaxBudgetEditor
|
||||
value={{}}
|
||||
onChange={onChange}
|
||||
availableModels={["gpt-4o", "claude-opus-4-8"]}
|
||||
premiumUser={true}
|
||||
/>,
|
||||
);
|
||||
|
||||
await user.click(addButton());
|
||||
const modelField = screen.getByPlaceholderText("Select model");
|
||||
await user.type(modelField, "claude-opus*");
|
||||
await user.click(await screen.findByRole("option", { name: /claude-opus\*/ }));
|
||||
fireEvent.change(screen.getByPlaceholderText("Max spend ($)"), { target: { value: "10" } });
|
||||
|
||||
expect(onChange).toHaveBeenLastCalledWith({ "claude-opus*": { budget_limit: 10, time_period: "30d" } });
|
||||
});
|
||||
|
||||
it("does not offer a typed pattern that another row already budgets", async () => {
|
||||
const user = userEvent.setup();
|
||||
renderWithProviders(
|
||||
<ModelMaxBudgetEditor
|
||||
value={{ "claude-opus*": { budget_limit: 10, time_period: "1d" } }}
|
||||
onChange={vi.fn()}
|
||||
availableModels={["gpt-4o", "claude-opus-4-8"]}
|
||||
premiumUser={true}
|
||||
/>,
|
||||
);
|
||||
|
||||
await user.click(addButton());
|
||||
const [, newRowModelField] = screen.getAllByPlaceholderText("Select model");
|
||||
await user.type(newRowModelField, "claude-opus*");
|
||||
|
||||
expect(screen.queryByRole("option", { name: /claude-opus\*/ })).not.toBeInTheDocument();
|
||||
await user.clear(newRowModelField);
|
||||
await user.type(newRowModelField, "claude-sonnet*");
|
||||
expect(await screen.findByRole("option", { name: /claude-sonnet\*/ })).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("does not offer a typed name that is neither listed nor a pattern", async () => {
|
||||
const user = userEvent.setup();
|
||||
renderWithProviders(
|
||||
<ModelMaxBudgetEditor
|
||||
value={{}}
|
||||
onChange={vi.fn()}
|
||||
availableModels={["gpt-4o", "claude-opus-4-8"]}
|
||||
premiumUser={true}
|
||||
/>,
|
||||
);
|
||||
|
||||
await user.click(addButton());
|
||||
await user.type(screen.getByPlaceholderText("Select model"), "claude-opsu-4-8");
|
||||
|
||||
expect(screen.queryByRole("option", { name: /claude-opsu-4-8/ })).not.toBeInTheDocument();
|
||||
expect(screen.getByText("No models found")).toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -50,6 +50,8 @@ export const MODEL_BUDGET_PERIOD_OPTIONS = [
|
|||
|
||||
const DEFAULT_PERIOD = "30d";
|
||||
|
||||
const MODEL_PATTERN_VALUE = { hint: "Model pattern", accepts: (typed: string) => typed.includes("*") };
|
||||
|
||||
export const entriesToModelMaxBudget = (entries: readonly ModelBudgetEntry[]): ModelMaxBudget =>
|
||||
Object.fromEntries(
|
||||
entries
|
||||
|
|
@ -121,7 +123,7 @@ export function ModelMaxBudgetEditor({
|
|||
const blurb = (
|
||||
<div className="text-xs text-muted-foreground">
|
||||
{premiumUser
|
||||
? "Cap spend per model over its own window. A budget set on the bare model name also covers the provider-prefixed spelling of that model."
|
||||
? "Cap spend per model over its own window. A budget set on the bare model name also covers the provider-prefixed spelling of that model. Type an access group or a pattern like claude-opus* to share one budget across every model it covers."
|
||||
: MODEL_MAX_BUDGET_PREMIUM_HINT}
|
||||
</div>
|
||||
);
|
||||
|
|
@ -165,6 +167,8 @@ export function ModelMaxBudgetEditor({
|
|||
onValueChange={(model) => updateEntry(entry.id, { model })}
|
||||
placeholder="Select model"
|
||||
emptyText="No models found"
|
||||
customValue={MODEL_PATTERN_VALUE}
|
||||
reservedValues={entries.flatMap((other) => (other.id !== entry.id && other.model ? [other.model] : []))}
|
||||
disabled={!premiumUser}
|
||||
/>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -63,6 +63,24 @@ describe("SearchSelect", () => {
|
|||
expect(onValueChange).toHaveBeenLastCalledWith("team-3");
|
||||
});
|
||||
|
||||
it("offers the typed text as a value only when the caller accepts it", async () => {
|
||||
const user = userEvent.setup();
|
||||
const onValueChange = vi.fn();
|
||||
const customValue = { hint: "Custom value", accepts: (typed: string) => typed.endsWith("*") };
|
||||
const { rerender } = render(<SearchSelect options={OPTIONS} onValueChange={onValueChange} />);
|
||||
const input = screen.getByRole("combobox");
|
||||
await user.click(input);
|
||||
fireEvent.change(input, { target: { value: "team-*" } });
|
||||
expect(screen.queryByRole("option", { name: /team-\*/ })).not.toBeInTheDocument();
|
||||
|
||||
rerender(<SearchSelect options={OPTIONS} onValueChange={onValueChange} customValue={customValue} />);
|
||||
fireEvent.change(input, { target: { value: "team-4" } });
|
||||
expect(screen.queryByRole("option", { name: /team-4/ })).not.toBeInTheDocument();
|
||||
fireEvent.change(input, { target: { value: "team-*" } });
|
||||
await user.click(await screen.findByRole("option", { name: /team-\*/ }));
|
||||
expect(onValueChange).toHaveBeenLastCalledWith("team-*");
|
||||
});
|
||||
|
||||
it("filters the options client-side as you type", async () => {
|
||||
const user = userEvent.setup();
|
||||
render(<SearchSelect options={OPTIONS} onValueChange={vi.fn()} />);
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
"use client";
|
||||
|
||||
import React from "react";
|
||||
import React, { useState } from "react";
|
||||
import {
|
||||
Combobox,
|
||||
ComboboxContent,
|
||||
|
|
@ -19,6 +19,11 @@ export interface SearchSelectOption {
|
|||
icon?: React.ReactNode;
|
||||
}
|
||||
|
||||
interface CustomValue {
|
||||
hint: string;
|
||||
accepts: (typed: string) => boolean;
|
||||
}
|
||||
|
||||
interface SearchSelectProps {
|
||||
options: SearchSelectOption[];
|
||||
value?: string | null;
|
||||
|
|
@ -29,6 +34,8 @@ interface SearchSelectProps {
|
|||
className?: string;
|
||||
inputId?: string;
|
||||
allowClear?: boolean;
|
||||
customValue?: CustomValue;
|
||||
reservedValues?: readonly string[];
|
||||
"aria-label"?: string;
|
||||
id?: string;
|
||||
}
|
||||
|
|
@ -49,13 +56,22 @@ export function SearchSelect({
|
|||
className,
|
||||
inputId,
|
||||
allowClear = true,
|
||||
customValue,
|
||||
reservedValues = [],
|
||||
"aria-label": ariaLabel,
|
||||
id,
|
||||
}: SearchSelectProps) {
|
||||
const [query, setQuery] = useState("");
|
||||
const selected =
|
||||
value == null || value === "" ? null : options.find((option) => option.value === value) ?? { label: value, value };
|
||||
const items =
|
||||
const known =
|
||||
selected !== null && !options.some((option) => option.value === selected.value) ? [selected, ...options] : options;
|
||||
const typed = query.trim();
|
||||
const isNewValue = typed !== "" && !reservedValues.includes(typed) && !known.some((option) => option.value === typed);
|
||||
const items =
|
||||
customValue !== undefined && isNewValue && customValue.accepts(typed)
|
||||
? [...known, { label: typed, value: typed, sublabel: customValue.hint }]
|
||||
: known;
|
||||
|
||||
return (
|
||||
<Combobox
|
||||
|
|
@ -65,6 +81,7 @@ export function SearchSelect({
|
|||
isItemEqualToValue={(a: SearchSelectOption, b: SearchSelectOption) => a.value === b.value}
|
||||
itemToStringLabel={(item: SearchSelectOption) => item.label}
|
||||
filter={matchesQuery}
|
||||
onInputValueChange={setQuery}
|
||||
disabled={disabled}
|
||||
>
|
||||
<ComboboxInput
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue