From 3d7347d22725380581b9160ba814fc79876a296d Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 18:21:59 -0700 Subject: [PATCH 1/6] feat(ui): let the per-model budget field take a model pattern or access group --- .../ModelMaxBudgetEditor.integration.test.tsx | 26 ++++++++++++++++++- .../key_team_helpers/ModelMaxBudgetEditor.tsx | 5 ++-- .../shared/SearchSelect.integration.test.tsx | 15 +++++++++++ .../src/components/shared/SearchSelect.tsx | 13 ++++++++-- 4 files changed, 54 insertions(+), 5 deletions(-) diff --git a/ui/litellm-dashboard/src/components/key_team_helpers/ModelMaxBudgetEditor.integration.test.tsx b/ui/litellm-dashboard/src/components/key_team_helpers/ModelMaxBudgetEditor.integration.test.tsx index 33f09092cca..c8fb108f66d 100644 --- a/ui/litellm-dashboard/src/components/key_team_helpers/ModelMaxBudgetEditor.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/key_team_helpers/ModelMaxBudgetEditor.integration.test.tsx @@ -1,5 +1,6 @@ +import userEvent from "@testing-library/user-event"; import { describe, expect, it, vi } from "vitest"; -import { renderWithProviders, screen } from "../../../tests/test-utils"; +import { fireEvent, renderWithProviders, screen } from "../../../tests/test-utils"; import { MODEL_MAX_BUDGET_PREMIUM_HINT, ModelMaxBudgetEditor, type ModelMaxBudget } from "./ModelMaxBudgetEditor"; const STORED: ModelMaxBudget = { "gpt-4o": { budget_limit: 5, time_period: "30d" } }; @@ -67,3 +68,26 @@ describe("ModelMaxBudgetEditor with an enterprise license", () => { expect(addButton()).toBeEnabled(); }); }); + +describe("ModelMaxBudgetEditor model families", () => { + it("stores a typed pattern that matches no listed model as one budget entry", async () => { + const user = userEvent.setup(); + const onChange = vi.fn(); + renderWithProviders( + , + ); + + await user.click(addButton()); + const modelField = screen.getByPlaceholderText("Select a model or type a pattern"); + await user.type(modelField, "claude-opus*"); + await user.click(await screen.findByRole("option", { name: /claude-opus\*/ })); + fireEvent.change(screen.getByPlaceholderText("Max spend ($)"), { target: { value: "10" } }); + + expect(onChange).toHaveBeenLastCalledWith({ "claude-opus*": { budget_limit: 10, time_period: "30d" } }); + }); +}); diff --git a/ui/litellm-dashboard/src/components/key_team_helpers/ModelMaxBudgetEditor.tsx b/ui/litellm-dashboard/src/components/key_team_helpers/ModelMaxBudgetEditor.tsx index a6e72068463..0fc0f2116be 100644 --- a/ui/litellm-dashboard/src/components/key_team_helpers/ModelMaxBudgetEditor.tsx +++ b/ui/litellm-dashboard/src/components/key_team_helpers/ModelMaxBudgetEditor.tsx @@ -121,7 +121,7 @@ export function ModelMaxBudgetEditor({ const blurb = (
{premiumUser - ? "Cap spend per model over its own window. A budget set on the bare model name also covers the provider-prefixed spelling of that model." + ? "Cap spend per model over its own window. A budget set on the bare model name also covers the provider-prefixed spelling of that model. Type an access group or a pattern like claude-opus* to share one budget across every model it covers." : MODEL_MAX_BUDGET_PREMIUM_HINT}
); @@ -163,8 +163,9 @@ export function ModelMaxBudgetEditor({ options={modelOptions.map((model) => ({ label: model, value: model }))} value={entry.model} onValueChange={(model) => updateEntry(entry.id, { model })} - placeholder="Select model" + placeholder="Select a model or type a pattern" emptyText="No models found" + customValueHint="Access group or model pattern" disabled={!premiumUser} /> diff --git a/ui/litellm-dashboard/src/components/shared/SearchSelect.integration.test.tsx b/ui/litellm-dashboard/src/components/shared/SearchSelect.integration.test.tsx index 9f320049b09..ce5b734941a 100644 --- a/ui/litellm-dashboard/src/components/shared/SearchSelect.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/shared/SearchSelect.integration.test.tsx @@ -63,6 +63,21 @@ describe("SearchSelect", () => { expect(onValueChange).toHaveBeenLastCalledWith("team-3"); }); + it("offers the typed text as a value only when the caller accepts custom values", async () => { + const user = userEvent.setup(); + const onValueChange = vi.fn(); + const { rerender } = render(); + const input = screen.getByRole("combobox"); + await user.click(input); + fireEvent.change(input, { target: { value: "team-*" } }); + expect(screen.queryByRole("option", { name: /team-\*/ })).not.toBeInTheDocument(); + + rerender(); + fireEvent.change(input, { target: { value: "team-*" } }); + await user.click(await screen.findByRole("option", { name: /team-\*/ })); + expect(onValueChange).toHaveBeenLastCalledWith("team-*"); + }); + it("filters the options client-side as you type", async () => { const user = userEvent.setup(); render(); diff --git a/ui/litellm-dashboard/src/components/shared/SearchSelect.tsx b/ui/litellm-dashboard/src/components/shared/SearchSelect.tsx index 3649114826d..343194dc046 100644 --- a/ui/litellm-dashboard/src/components/shared/SearchSelect.tsx +++ b/ui/litellm-dashboard/src/components/shared/SearchSelect.tsx @@ -1,6 +1,6 @@ "use client"; -import React from "react"; +import React, { useState } from "react"; import { Combobox, ComboboxContent, @@ -29,6 +29,7 @@ interface SearchSelectProps { className?: string; inputId?: string; allowClear?: boolean; + customValueHint?: string; "aria-label"?: string; id?: string; } @@ -49,13 +50,20 @@ export function SearchSelect({ className, inputId, allowClear = true, + customValueHint, "aria-label": ariaLabel, id, }: SearchSelectProps) { + const [query, setQuery] = useState(""); const selected = value == null || value === "" ? null : options.find((option) => option.value === value) ?? { label: value, value }; - const items = + const known = selected !== null && !options.some((option) => option.value === selected.value) ? [selected, ...options] : options; + const typed = query.trim(); + const items = + customValueHint !== undefined && typed !== "" && !known.some((option) => option.value === typed) + ? [...known, { label: typed, value: typed, sublabel: customValueHint }] + : known; return ( a.value === b.value} itemToStringLabel={(item: SearchSelectOption) => item.label} filter={matchesQuery} + onInputValueChange={setQuery} disabled={disabled} > Date: Tue, 20 Oct 2026 19:08:13 -0700 Subject: [PATCH 2/6] feat(proxy): let model_max_budget entries name a model family or access group, shared across replicas --- .../hooks/model_budget_shared_windows.py | 128 +++++++ .../proxy/hooks/model_max_budget_limiter.py | 313 +++++++++++++++--- .../hooks/test_model_budget_shared_windows.py | 93 ++++++ .../hooks/test_model_max_budget_limiter.py | 257 +++++++++++++- 4 files changed, 729 insertions(+), 62 deletions(-) create mode 100644 litellm/proxy/hooks/model_budget_shared_windows.py create mode 100644 tests/unit/proxy/hooks/test_model_budget_shared_windows.py diff --git a/litellm/proxy/hooks/model_budget_shared_windows.py b/litellm/proxy/hooks/model_budget_shared_windows.py new file mode 100644 index 00000000000..344114edaa4 --- /dev/null +++ b/litellm/proxy/hooks/model_budget_shared_windows.py @@ -0,0 +1,128 @@ +from collections.abc import Awaitable, Callable +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Final, Protocol, TypeAlias + +from litellm._logging import verbose_proxy_logger +from litellm.caching.in_memory_cache import InMemoryCache +from litellm.proxy.db.db_transaction_queue.window_spend_update_queue import ( + WindowSpendTransaction, + build_window_spend_transaction, +) +from litellm.repositories.table_repositories import BudgetWindowSpendRepository +from litellm.router_strategy.budget_limiter import budget_window_seconds + +if TYPE_CHECKING: + from litellm.proxy.utils import PrismaClient + +MODEL_BUDGET_WINDOW_ENTITY_TYPE: Final = "model_max_budget" +_READ_TTL_SECONDS: Final = 5 + + +@dataclass(frozen=True, slots=True) +class SharedWindow: + started_at: float + spend: float + + def is_open(self, budget_duration: str, now: float) -> bool: + return now < self.started_at + budget_window_seconds(self.started_at, budget_duration) + + +class LocalTTLCache(Protocol): + def get_cache(self, key: str) -> object: ... + + def set_cache(self, key: str, value: object, *, ttl: float) -> None: ... + + +class SharedWindows(Protocol): + async def read(self, spend_key: str, budget_duration: str) -> SharedWindow | None: ... + + async def add(self, spend_key: str, budget_duration: str, started_at: float, spend: float) -> None: ... + + +PrismaClientSource: TypeAlias = Callable[[], "PrismaClient | None"] +WindowSpendSink: TypeAlias = Callable[[WindowSpendTransaction], Awaitable[None]] + + +def _proxy_prisma_client() -> "PrismaClient | None": + try: + from litellm.proxy.proxy_server import prisma_client + except ImportError: + return None + return prisma_client + + +async def _queue_on_proxy_spend_writer(transaction: WindowSpendTransaction) -> None: + from litellm.proxy.proxy_server import proxy_logging_obj + + await proxy_logging_obj.db_spend_update_writer.window_spend_update_queue.add_update(transaction) + + +@dataclass(frozen=True, slots=True) +class _Reading: + window: SharedWindow | None + + +class DatabaseSharedWindows: + def __init__( + self, + prisma_client_source: PrismaClientSource = _proxy_prisma_client, + sink: WindowSpendSink = _queue_on_proxy_spend_writer, + read_ttl_seconds: int = _READ_TTL_SECONDS, + ) -> None: + self._prisma_client_source = prisma_client_source + self._sink = sink + self._read_ttl_seconds = read_ttl_seconds + self._readings: LocalTTLCache = InMemoryCache(max_size_in_memory=10_000, default_ttl=read_ttl_seconds) + + async def read(self, spend_key: str, budget_duration: str) -> SharedWindow | None: + cached: Final = self._readings.get_cache(key=spend_key) + if isinstance(cached, _Reading): + return cached.window + window: Final = await self._read_row(spend_key=spend_key, budget_duration=budget_duration) + self._readings.set_cache(key=spend_key, value=_Reading(window=window), ttl=self._read_ttl_seconds) + return window + + async def add(self, spend_key: str, budget_duration: str, started_at: float, spend: float) -> None: + if self._prisma_client_source() is None: + return + try: + await self._sink( + build_window_spend_transaction( + entity_type=MODEL_BUDGET_WINDOW_ENTITY_TYPE, + entity_id=spend_key, + window_duration=budget_duration, + window_start=datetime.fromtimestamp(started_at, tz=timezone.utc), + spend=spend, + ) + ) + except Exception: # noqa: BLE001 # spend tracking must never fail the success callback + verbose_proxy_logger.debug("Unable to queue shared model budget spend for %s", spend_key, exc_info=True) + + async def _read_row(self, spend_key: str, budget_duration: str) -> SharedWindow | None: + prisma_client: Final = self._prisma_client_source() + if prisma_client is None: + return None + try: + row: Final = await BudgetWindowSpendRepository(prisma_client).table.find_unique( + where={ + "entity_type_entity_id_window_duration": { + "entity_type": MODEL_BUDGET_WINDOW_ENTITY_TYPE, + "entity_id": spend_key, + "window_duration": budget_duration, + } + } + ) + except Exception: # noqa: BLE001 # an unreadable row leaves enforcement on this replica's own counter + verbose_proxy_logger.exception("Unable to read shared model budget spend for %s", spend_key) + return None + if row is None: + return None + window_start: Final = row.window_start + started_at: Final = ( + window_start if window_start.tzinfo is not None else window_start.replace(tzinfo=timezone.utc) + ).timestamp() + return SharedWindow(started_at=started_at, spend=float(row.spend or 0.0)) + + +PROXY_SHARED_WINDOWS: Final = DatabaseSharedWindows() diff --git a/litellm/proxy/hooks/model_max_budget_limiter.py b/litellm/proxy/hooks/model_max_budget_limiter.py index 37cef41034b..d742e8148a7 100644 --- a/litellm/proxy/hooks/model_max_budget_limiter.py +++ b/litellm/proxy/hooks/model_max_budget_limiter.py @@ -1,11 +1,13 @@ import asyncio +import functools import json +import re import time -from collections.abc import Callable, Iterable, Mapping, Sequence +from collections.abc import Callable, Collection, Iterable, Iterator, Mapping, Sequence from dataclasses import dataclass from datetime import datetime, timezone from types import MappingProxyType -from typing import Final, cast # noqa: TID251 # untyped callback payload requires explicit narrowing +from typing import Final, TypeAlias, cast # noqa: TID251 # untyped callback payload requires explicit narrowing import litellm from litellm._internal_context import with_service_target @@ -16,8 +18,15 @@ from litellm.litellm_core_utils.duration_parser import duration_in_seconds from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup from litellm.llms.bedrock.common_utils import get_bedrock_base_model from litellm.proxy._types import Litellm_EntityType, UserAPIKeyAuth +from litellm.proxy.hooks.model_budget_shared_windows import ( + PROXY_SHARED_WINDOWS, + LocalTTLCache, + SharedWindow, + SharedWindows, +) from litellm.router_strategy.budget_limiter import RouterBudgetLimiting, budget_window_seconds from litellm.router_utils.batch_utils import batch_charged_once_marker_key, batch_id_to_charge_once +from litellm.router_utils.pattern_match_deployments import PatternUtils from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import BudgetConfig, StandardLoggingPayload @@ -43,6 +52,10 @@ _PROCESS_STARTED_AT: Final = time.monotonic() _LATEST_RESET_AT: Final = datetime(9999, 12, 31, tzinfo=timezone.utc).timestamp() +_GLOB_WILDCARD: Final = "*" + +ModelAccessGroups: TypeAlias = Callable[[str], Collection[str]] + _BUDGET_START_TIME_KEY_PREFIXES: Final = MappingProxyType( { Litellm_EntityType.KEY: "virtual_key_budget_start_time", @@ -131,36 +144,100 @@ def model_budget_start_time_cache_key( return f"{_BUDGET_START_TIME_KEY_PREFIXES[entity_type]}:{entity_id}:{budget_model}:{budget_duration}" -def resolve_model_budget(model: str, model_max_budget: Mapping[str, object]) -> ResolvedModelBudget | None: - """Find the `model_max_budget` entry that governs `model`, or None.""" - for candidate in _budget_model_candidates(model): - raw_budget_config = model_max_budget.get(candidate) - if raw_budget_config is None: - continue - if (budget_config := _usable_budget_config(raw_budget_config)) is None: - # An entry that will not validate cannot be keyed, so it cannot be - # enforced or incremented. Skip to the next candidate rather than - # raising: raising would abort every other scope's increment and turn - # a config typo into a 500, and stopping here would let one malformed - # specific entry disable a perfectly good bare-family budget beside - # it. The candidate chain already falls through an ABSENT entry, and - # an unparseable one is indistinguishable from absent to enforcement. - # `validate_model_max_budget` rejects these on the write path, so - # reaching here means config.yaml or a direct DB edit. - verbose_proxy_logger.warning( - "Ignoring unusable model_max_budget entry for %s; it cannot be enforced or tracked", - candidate, - ) - continue - return ResolvedModelBudget(budget_model=candidate, budget_config=budget_config) - return None +def proxy_model_access_groups(model: str) -> frozenset[str]: + try: + from litellm.proxy.proxy_server import llm_router + except ImportError: + return frozenset() + if llm_router is None: + return frozenset() + return frozenset(llm_router.get_model_access_groups(model_name=model)) -def team_model_budget_applies(model: str, key_model_max_budget: Mapping[str, object] | None) -> bool: +def resolve_model_budget( + model: str, + model_max_budget: Mapping[str, object], + model_access_groups: ModelAccessGroups = proxy_model_access_groups, +) -> ResolvedModelBudget | None: + """Find the `model_max_budget` entry that governs `model`, or None. + + An entry named after the model wins, then one named after a model access + group serving it, then the most specific glob (`claude-opus*`). Every model + an access group or glob entry covers draws from that entry's one counter. + """ + resolved: Final = ( + _usable_entry(entry=entry, raw_budget_config=model_max_budget[entry]) + for entry in _governing_entry_names( + model=model, model_max_budget=model_max_budget, model_access_groups=model_access_groups + ) + ) + return next((entry for entry in resolved if entry is not None), None) + + +def _governing_entry_names( + model: str, + model_max_budget: Mapping[str, object], + model_access_groups: ModelAccessGroups, +) -> Iterator[str]: + candidates: Final = _budget_model_candidates(model) + yield from (candidate for candidate in candidates if candidate in model_max_budget) + group_entries: Final = tuple( + sorted(entry for entry in model_max_budget if _GLOB_WILDCARD not in entry and entry not in candidates) + ) + if group_entries: + groups: Final = model_access_groups(model) + yield from (entry for entry in group_entries if entry in groups) + yield from sorted( + (entry for entry in model_max_budget if _GLOB_WILDCARD in entry and _glob_matches(entry, candidates)), + key=_glob_precedence, + reverse=True, + ) + + +def _glob_matches(pattern: str, names: Iterable[str]) -> bool: + regex: Final = _glob_regex(pattern) + return any(regex.fullmatch(name) for name in names) + + +@functools.lru_cache(maxsize=1024) +def _glob_regex(pattern: str) -> re.Pattern[str]: + return re.compile(re.escape(pattern).replace(re.escape(_GLOB_WILDCARD), ".*")) + + +def _glob_precedence(pattern: str) -> tuple[tuple[int, int], str]: + return PatternUtils.calculate_pattern_specificity(pattern), pattern + + +def _usable_entry(entry: str, raw_budget_config: object) -> ResolvedModelBudget | None: + if (budget_config := _usable_budget_config(raw_budget_config)) is None: + # An entry that will not validate cannot be keyed, so it cannot be + # enforced or incremented. Skip to the next candidate rather than + # raising: raising would abort every other scope's increment and turn + # a config typo into a 500, and stopping here would let one malformed + # specific entry disable a perfectly good bare-family budget beside + # it. The candidate chain already falls through an ABSENT entry, and + # an unparseable one is indistinguishable from absent to enforcement. + # `validate_model_max_budget` rejects these on the write path, so + # reaching here means config.yaml or a direct DB edit. + verbose_proxy_logger.warning( + "Ignoring unusable model_max_budget entry for %s; it cannot be enforced or tracked", + entry, + ) + return None + return ResolvedModelBudget(budget_model=entry, budget_config=budget_config) + + +def team_model_budget_applies( + model: str, + key_model_max_budget: Mapping[str, object] | None, + model_access_groups: ModelAccessGroups = proxy_model_access_groups, +) -> bool: """A key entry that spend-gates `model` overrides the team cap: it is then gated on and billed to the key alone.""" if not key_model_max_budget: return True - resolved: Final = resolve_model_budget(model=model, model_max_budget=key_model_max_budget) + resolved: Final = resolve_model_budget( + model=model, model_max_budget=key_model_max_budget, model_access_groups=model_access_groups + ) return resolved is None or not _spend_gated(resolved.budget_config) @@ -208,6 +285,7 @@ async def build_model_max_budget_usage( model_max_budget: Mapping[str, object] | None, cache: DualCache | None, now: datetime | None = None, + shared_windows: SharedWindows = PROXY_SHARED_WINDOWS, ) -> dict[str, dict[str, object]]: """Current-window spend per configured budget model, as `/key/info` and `/user/info` report it. @@ -231,17 +309,18 @@ async def build_model_max_budget_usage( ) if not budgets: return {} + spend_keys: Final = tuple( + model_budget_spend_cache_key( + entity_type=entity_type, + entity_id=entity_id, + budget_model=budget_model, + budget_duration=budget_config.budget_duration, + ) + for budget_model, budget_config in budgets + ) windows: Final = await _read_windows( cache=cache, - spend_keys=tuple( - model_budget_spend_cache_key( - entity_type=entity_type, - entity_id=entity_id, - budget_model=budget_model, - budget_duration=budget_config.budget_duration, - ) - for budget_model, budget_config in budgets - ), + spend_keys=spend_keys, start_time_keys=tuple( model_budget_start_time_cache_key( entity_type=entity_type, @@ -253,21 +332,57 @@ async def build_model_max_budget_usage( ), ) current_time: Final = (now or datetime.now(timezone.utc)).timestamp() + shared: Final = tuple( + [ + ( + await _open_shared_window( + shared_windows=shared_windows, + spend_key=spend_key, + budget_duration=budget_config.budget_duration, + now=current_time, + ) + if cache.redis_cache is None + else None + ) + for spend_key, (_, budget_config) in zip(spend_keys, budgets, strict=True) + ] + ) + readings: Final = tuple( + _fuller_reading(local=window, shared=shared_window) + for window, shared_window in zip(windows, shared, strict=True) + ) return { budget_model: { - "current_spend": round(window.current_spend, 4), + "current_spend": round(reading.current_spend, 4), "budget_limit": budget_config.max_budget, "time_period": budget_config.budget_duration, "budget_reset_at": _window_reset_at( - started_at=window.started_at, + started_at=reading.started_at, budget_duration=budget_config.budget_duration, now=current_time, ), } - for (budget_model, budget_config), window in zip(budgets, windows, strict=True) + for (budget_model, budget_config), reading in zip(budgets, readings, strict=True) } +async def _open_shared_window( + shared_windows: SharedWindows, spend_key: str, budget_duration: str | None, now: float +) -> SharedWindow | None: + if budget_duration is None: + return None + window: Final = await shared_windows.read(spend_key=spend_key, budget_duration=budget_duration) + if window is None or not window.is_open(budget_duration=budget_duration, now=now): + return None + return window + + +def _fuller_reading(local: _WindowReading, shared: SharedWindow | None) -> _WindowReading: + if shared is None or shared.spend <= local.current_spend: + return local + return _WindowReading(current_spend=shared.spend, started_at=shared.started_at) + + @with_service_target("model_budgets") async def _read_windows( cache: DualCache, spend_keys: Sequence[str], start_time_keys: Sequence[str] @@ -337,13 +452,18 @@ def _as_float(value: object) -> float | None: def _resolve_entity_model_budgets( model: str, entity_budgets: Iterable[tuple[Litellm_EntityType, str | None, object]], + model_access_groups: ModelAccessGroups, ) -> tuple[tuple[Litellm_EntityType, str, ResolvedModelBudget], ...]: """Drop the scopes that do not budget `model`, keeping only what can be incremented.""" return tuple( (entity_type, entity_id, resolved) for entity_type, entity_id, model_max_budget in entity_budgets if entity_id is not None and isinstance(model_max_budget, Mapping) and model_max_budget - for resolved in (resolve_model_budget(model=model, model_max_budget=model_max_budget),) + for resolved in ( + resolve_model_budget( + model=model, model_max_budget=model_max_budget, model_access_groups=model_access_groups + ), + ) if resolved is not None and resolved.budget_config.budget_duration is not None ) @@ -367,9 +487,17 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting): Example: key=$LITELLM_API_KEY, model=gpt-4o, max_budget=100, time_period=1d """ - def __init__(self, dual_cache: DualCache, clock: Callable[[], float] = time.time): + def __init__( + self, + dual_cache: DualCache, + clock: Callable[[], float] = time.time, + model_access_groups: ModelAccessGroups = proxy_model_access_groups, + shared_windows: SharedWindows = PROXY_SHARED_WINDOWS, + ): self.dual_cache = dual_cache self._clock = clock + self._model_access_groups = model_access_groups + self._shared_windows = shared_windows self.redis_increment_operation_queue = [] self._redis_increment_queue_lock = asyncio.Lock() self._redis_increment_flush_lock = asyncio.Lock() @@ -471,7 +599,9 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting): Raises: BudgetExceededError: If the team has exceeded the model budget """ - if not team_model_budget_applies(model=model, key_model_max_budget=key_model_max_budget): + if not team_model_budget_applies( + model=model, key_model_max_budget=key_model_max_budget, model_access_groups=self._model_access_groups + ): return True return await self._is_entity_within_model_budget( entity_type=Litellm_EntityType.TEAM, @@ -509,7 +639,9 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting): ) -> bool: if not model_max_budget: return True - resolved: Final = resolve_model_budget(model=model, model_max_budget=model_max_budget) + resolved: Final = resolve_model_budget( + model=model, model_max_budget=model_max_budget, model_access_groups=self._model_access_groups + ) if resolved is None: verbose_proxy_logger.debug("Model %s not found in %s model_max_budget", model, entity_type.value) return True @@ -523,6 +655,7 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting): entity_id=entity_id, model=model, resolved=resolved, + max_budget=max_budget, ) if current_spend >= max_budget: raise litellm.BudgetExceededError( @@ -540,12 +673,16 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting): entity_id: str | None, model: str, resolved: ResolvedModelBudget, + max_budget: float, ) -> float: """Spend charged to this budget in the current window, legacy counter included. A counter that was never written is zero spend, not unknown spend. The distinction only shows up at a zero-dollar cap, where skipping the comparison would let the strictest possible limit admit every request. + + Without Redis each replica's counter holds only its own share, so the + window every replica adds to in the database is the floor. """ spend_key: Final = model_budget_spend_cache_key( entity_type=entity_type, @@ -559,10 +696,21 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting): model=model, resolved=resolved, ) - current_spend: Final = _as_spend(await self._cached_spend(spend_key)) - if legacy_spend_key is None or legacy_spend_key == spend_key: - return current_spend - return current_spend + _as_spend(await self._cached_spend(legacy_spend_key)) + counted: Final = _as_spend(await self._cached_spend(spend_key)) + local_spend: Final = ( + counted + if legacy_spend_key is None or legacy_spend_key == spend_key + else counted + _as_spend(await self._cached_spend(legacy_spend_key)) + ) + if local_spend >= max_budget or self.dual_cache.redis_cache is not None: + return local_spend + shared: Final = await _open_shared_window( + shared_windows=self._shared_windows, + spend_key=spend_key, + budget_duration=resolved.budget_config.budget_duration, + now=self._clock(), + ) + return local_spend if shared is None else max(local_spend, shared.spend) async def _cached_spend(self, spend_key: str) -> float | None: redis_cache: Final = self.dual_cache.redis_cache @@ -643,6 +791,7 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting): key_model_max_budget=( key_model_max_budget if isinstance(key_model_max_budget, Mapping) else None ), + model_access_groups=self._model_access_groups, ) else None ), @@ -667,7 +816,9 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting): ), ) - resolved_budgets: Final = _resolve_entity_model_budgets(model=model, entity_budgets=entity_budgets) + resolved_budgets: Final = _resolve_entity_model_budgets( + model=model, entity_budgets=entity_budgets, model_access_groups=self._model_access_groups + ) if not resolved_budgets: verbose_proxy_logger.debug( "Not running _PROXY_VirtualKeyModelMaxBudgetLimiter.async_log_success_event: " @@ -721,17 +872,61 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting): ttl_seconds=duration_in_seconds(budget_duration), ): return + start_time_key: Final = model_budget_start_time_cache_key( + entity_type=entity_type, + entity_id=entity_id, + budget_model=resolved.budget_model, + budget_duration=budget_duration, + ) await self._increment_spend_for_key( budget_config=resolved.budget_config, spend_key=spend_key, - start_time_key=model_budget_start_time_cache_key( - entity_type=entity_type, - entity_id=entity_id, - budget_model=resolved.budget_model, - budget_duration=budget_duration, - ), + start_time_key=start_time_key, response_cost=response_cost, ) + await self._share_spend( + spend_key=spend_key, + start_time_key=start_time_key, + budget_duration=budget_duration, + response_cost=response_cost, + ) + + async def _share_spend( + self, spend_key: str, start_time_key: str, budget_duration: str, response_cost: float + ) -> None: + """Without Redis, add this charge to the window every replica reads. + + A replica joins the window that is already open rather than its own, so + all of them roll over together and none resets the others' spend. + """ + if self.dual_cache.redis_cache is not None: + return + now: Final = self._clock() + shared: Final = await _open_shared_window( + shared_windows=self._shared_windows, + spend_key=spend_key, + budget_duration=budget_duration, + now=now, + ) + if shared is not None: + _join_window( + local=self.dual_cache.in_memory_cache, + spend_key=spend_key, + start_time_key=start_time_key, + window=shared, + budget_duration=budget_duration, + now=now, + ) + await self._shared_windows.add( + spend_key=spend_key, + budget_duration=budget_duration, + started_at=await self._get_or_set_budget_start_time( + start_time_key=start_time_key, + current_time=now, + ttl_seconds=budget_window_seconds(now, budget_duration), + ), + spend=response_cost, + ) async def _claim_batch_charge(self, spend_key: str, batch_id: str, ttl_seconds: int) -> bool: marker_key: Final = batch_charged_once_marker_key(spend_key=spend_key, batch_id=batch_id) @@ -741,4 +936,14 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting): return polls == 1 +def _join_window( + local: LocalTTLCache, spend_key: str, start_time_key: str, window: SharedWindow, budget_duration: str, now: float +) -> None: + remaining: Final = window.started_at + budget_window_seconds(window.started_at, budget_duration) - now + local.set_cache(key=start_time_key, value=window.started_at, ttl=remaining) + counted: Final = local.get_cache(key=spend_key) + if counted is not None: + local.set_cache(key=spend_key, value=counted, ttl=remaining) + + PROXY_VirtualKeyModelMaxBudgetLimiter = _PROXY_VirtualKeyModelMaxBudgetLimiter diff --git a/tests/unit/proxy/hooks/test_model_budget_shared_windows.py b/tests/unit/proxy/hooks/test_model_budget_shared_windows.py new file mode 100644 index 00000000000..70957b1f74d --- /dev/null +++ b/tests/unit/proxy/hooks/test_model_budget_shared_windows.py @@ -0,0 +1,93 @@ +from datetime import datetime, timezone +from types import SimpleNamespace +from typing import Final + +import pytest + +from litellm.proxy.db.db_transaction_queue.window_spend_update_queue import WindowSpendTransaction +from litellm.proxy.hooks.model_budget_shared_windows import DatabaseSharedWindows, SharedWindow + +SPEND_KEY: Final = "virtual_key_spend:key-hash:claude-opus*:1d" +WINDOW_START: Final = datetime(2026, 10, 20, 9, 30, tzinfo=timezone.utc) + + +class _Table: + def __init__(self, row: object | None = None, error: Exception | None = None) -> None: + self.row = row + self.error = error + self.reads = 0 + + async def find_unique(self, where: object) -> object | None: + self.reads = self.reads + 1 + if self.error is not None: + raise self.error + return self.row + + +class _Sink: + def __init__(self) -> None: + self.transactions: tuple[WindowSpendTransaction, ...] = () + + async def __call__(self, transaction: WindowSpendTransaction) -> None: + self.transactions = (*self.transactions, transaction) + + +def _database_windows(table: _Table | None, sink: _Sink) -> DatabaseSharedWindows: + prisma: Final = None if table is None else SimpleNamespace(db=SimpleNamespace(litellm_budgetwindowspend=table)) + return DatabaseSharedWindows( + prisma_client_source=lambda: prisma, # pyright: ignore[reportArgumentType] # duck-typed prisma double + sink=sink, + ) + + +def test_a_window_is_open_until_its_period_has_passed(): + window: Final = SharedWindow(started_at=WINDOW_START.timestamp(), spend=1.0) + day: Final = 24 * 60 * 60 + + assert window.is_open(budget_duration="1d", now=WINDOW_START.timestamp() + day - 1) + assert not window.is_open(budget_duration="1d", now=WINDOW_START.timestamp() + day) + + +@pytest.mark.asyncio +async def test_read_returns_the_stored_window_and_rereads_only_after_the_cache_expires(): + table: Final = _Table(row=SimpleNamespace(window_start=WINDOW_START.replace(tzinfo=None), spend=2.5)) + windows: Final = _database_windows(table, _Sink()) + + first: Final = await windows.read(spend_key=SPEND_KEY, budget_duration="1d") + second: Final = await windows.read(spend_key=SPEND_KEY, budget_duration="1d") + + assert first == SharedWindow(started_at=WINDOW_START.timestamp(), spend=2.5) + assert second == first + assert table.reads == 1 + + +@pytest.mark.asyncio +async def test_an_unreadable_table_reads_as_no_shared_window(): + table: Final = _Table(error=RuntimeError("database is down")) + windows: Final = _database_windows(table, _Sink()) + + assert await windows.read(spend_key=SPEND_KEY, budget_duration="1d") is None + + +@pytest.mark.asyncio +async def test_add_queues_the_charge_for_the_window_it_belongs_to(): + sink: Final = _Sink() + windows: Final = _database_windows(_Table(), sink) + + await windows.add(spend_key=SPEND_KEY, budget_duration="1d", started_at=WINDOW_START.timestamp(), spend=0.25) + + assert len(sink.transactions) == 1 + queued: Final = sink.transactions[0] + assert (queued["entity_id"], queued["window_duration"], queued["spend"]) == (SPEND_KEY, "1d", 0.25) + assert datetime.fromisoformat(queued["window_start"]).replace(tzinfo=timezone.utc) == WINDOW_START + + +@pytest.mark.asyncio +async def test_nothing_is_queued_without_a_database(): + sink: Final = _Sink() + windows: Final = _database_windows(None, sink) + + await windows.add(spend_key=SPEND_KEY, budget_duration="1d", started_at=WINDOW_START.timestamp(), spend=0.25) + + assert sink.transactions == () + assert await windows.read(spend_key=SPEND_KEY, budget_duration="1d") is None diff --git a/tests/unit/proxy/hooks/test_model_max_budget_limiter.py b/tests/unit/proxy/hooks/test_model_max_budget_limiter.py index 2678f4205d5..33fa7bc3daf 100644 --- a/tests/unit/proxy/hooks/test_model_max_budget_limiter.py +++ b/tests/unit/proxy/hooks/test_model_max_budget_limiter.py @@ -8,10 +8,13 @@ import pytest import litellm from litellm.caching.caching import DualCache -from litellm.proxy._types import Litellm_EntityType +from litellm.proxy._types import Litellm_EntityType, UserAPIKeyAuth +from litellm.proxy.hooks.model_budget_shared_windows import SharedWindow, SharedWindows from litellm.proxy.hooks.model_max_budget_limiter import ( PROXY_VirtualKeyModelMaxBudgetLimiter, + build_model_max_budget_usage, model_budget_spend_cache_key, + resolve_model_budget, team_member_budget_entity_id, ) from litellm.types.caching import RedisPipelineIncrementOperation @@ -74,10 +77,14 @@ def _event( } +async def _log_success( + limiter: PROXY_VirtualKeyModelMaxBudgetLimiter, event: dict[str, object], response_obj: object = None +) -> None: + await limiter.async_log_success_event(event, response_obj=response_obj, start_time=None, end_time=None) + + async def _poll(limiter: PROXY_VirtualKeyModelMaxBudgetLimiter, batch: LiteLLMBatch, response_cost: float) -> None: - await limiter.async_log_success_event( - _event("aretrieve_batch", response_cost), response_obj=batch, start_time=None, end_time=None - ) + await _log_success(limiter, _event("aretrieve_batch", response_cost), response_obj=batch) async def _chat( @@ -86,7 +93,8 @@ async def _chat( user_id: str = USER_ID, team_id: str = TEAM_ID, ) -> None: - await limiter.async_log_success_event( + await _log_success( + limiter, _event( "acompletion", CHAT_COST, @@ -94,9 +102,6 @@ async def _chat( user_id=user_id, team_id=team_id, ), - response_obj=None, - start_time=None, - end_time=None, ) @@ -311,3 +316,239 @@ async def test_a_batch_polled_within_every_budget_window_is_never_charged_again( await _poll(limiter, finished, BATCH_COST) assert _local_spend(limiter, KEY_SPEND_KEY) == pytest.approx(BATCH_COST) + + +OPUS_GROUP: Final = "claude-opus" +OPUS_GROUP_MEMBERS: Final = MappingProxyType( + {"claude-opus-4-8": frozenset({OPUS_GROUP}), "claude-opus-5-5": frozenset({OPUS_GROUP})} +) +FAMILY_CAP: Final = {"budget_limit": CHAT_COST, "time_period": "1d"} # mutable-ok: BudgetConfig validates dicts only +ROOMY_CAP: Final = {"budget_limit": CHAT_COST * 100, "time_period": "1d"} # mutable-ok: BudgetConfig validates dicts only + + +def _model_access_groups(groups_by_model: Mapping[str, frozenset[str]]) -> Callable[[str], frozenset[str]]: + return lambda model: groups_by_model.get(model, frozenset()) + + +class _SharedWindowsDouble: + def __init__(self) -> None: + self.rows: Mapping[str, SharedWindow] = MappingProxyType({}) + + async def read(self, spend_key: str, budget_duration: str) -> SharedWindow | None: + return self.rows.get(spend_key) + + async def add(self, spend_key: str, budget_duration: str, started_at: float, spend: float) -> None: + stored: Final = self.rows.get(spend_key) + row: Final = ( + SharedWindow(started_at=stored.started_at, spend=stored.spend + spend) + if stored is not None and stored.started_at >= started_at + else SharedWindow(started_at=started_at, spend=spend) + ) + self.rows = MappingProxyType({**self.rows, spend_key: row}) + + +def _family_limiter( + groups_by_model: Mapping[str, frozenset[str]] = OPUS_GROUP_MEMBERS, + dual_cache: DualCache | None = None, + shared_windows: SharedWindows | None = None, + clock: Callable[[], float] = time.time, +) -> PROXY_VirtualKeyModelMaxBudgetLimiter: + return PROXY_VirtualKeyModelMaxBudgetLimiter( + dual_cache=dual_cache or DualCache(), + clock=clock, + model_access_groups=_model_access_groups(groups_by_model), + shared_windows=shared_windows or _SharedWindowsDouble(), + ) + + +def _key(model_max_budget: Mapping[str, object]) -> UserAPIKeyAuth: + return UserAPIKeyAuth(token=KEY_HASH, key_alias="opus-family", model_max_budget=dict(model_max_budget)) + + +async def _spend_on( + limiter: PROXY_VirtualKeyModelMaxBudgetLimiter, model: str, model_max_budget: Mapping[str, object] +) -> None: + await _log_success( + limiter, + { + "call_type": "acompletion", + "standard_logging_object": { + "call_type": "acompletion", + "response_cost": CHAT_COST, + "model": f"anthropic/{model}", + "model_group": model, + "metadata": {"user_api_key_hash": KEY_HASH}, + }, + "litellm_params": {"metadata": {"user_api_key_model_max_budget": dict(model_max_budget)}}, + }, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("family_entry", ["claude-opus*", OPUS_GROUP]) +async def test_every_opus_version_draws_from_one_family_entry(family_entry: str): + budget: Final = {family_entry: FAMILY_CAP} + limiter: Final = _family_limiter() + assert await limiter.is_key_within_model_budget(user_api_key_dict=_key(budget), model="claude-opus-5-5") + + await _spend_on(limiter, model="claude-opus-4-8", model_max_budget=budget) + + with pytest.raises(litellm.BudgetExceededError, match="model=claude-opus-5-5"): + await limiter.is_key_within_model_budget(user_api_key_dict=_key(budget), model="claude-opus-5-5") + assert await limiter.is_key_within_model_budget(user_api_key_dict=_key(budget), model="claude-sonnet-5") + + +@pytest.mark.asyncio +async def test_a_model_added_to_the_access_group_later_is_covered_with_no_key_change(): + budget: Final = {OPUS_GROUP: FAMILY_CAP} + shared_cache: Final = DualCache() + before_group_change: Final = _family_limiter(dual_cache=shared_cache) + after_group_change: Final = _family_limiter( + groups_by_model={**OPUS_GROUP_MEMBERS, "claude-opus-6": frozenset({OPUS_GROUP})}, dual_cache=shared_cache + ) + + await _spend_on(before_group_change, model="claude-opus-4-8", model_max_budget=budget) + + assert await before_group_change.is_key_within_model_budget(user_api_key_dict=_key(budget), model="claude-opus-6") + with pytest.raises(litellm.BudgetExceededError): + await after_group_change.is_key_within_model_budget(user_api_key_dict=_key(budget), model="claude-opus-6") + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("budget", "checked_model", "blocked"), + [ + ({"claude-opus-4-8": ROOMY_CAP, "claude-opus*": FAMILY_CAP}, "claude-opus-4-8", False), + ({"claude-opus-4-8": ROOMY_CAP, "claude-opus*": FAMILY_CAP}, "claude-opus-5-5", False), + ({OPUS_GROUP: FAMILY_CAP, "claude-opus*": ROOMY_CAP}, "claude-opus-5-5", True), + ({"claude-opus-4*": FAMILY_CAP, "claude-opus*": ROOMY_CAP}, "claude-opus-4-7", True), + ({"claude-opus-4*": FAMILY_CAP, "claude-opus*": ROOMY_CAP}, "claude-opus-5-5", False), + ({"claude-opus-4*": {"budget_limit": CHAT_COST}, "claude-opus*": FAMILY_CAP}, "claude-opus-5-5", True), + ], +) +async def test_the_most_specific_entry_takes_the_spend_and_gates_the_request( + budget: Mapping[str, object], checked_model: str, blocked: bool +): + limiter: Final = _family_limiter() + + await _spend_on(limiter, model="claude-opus-4-8", model_max_budget=budget) + + if blocked: + with pytest.raises(litellm.BudgetExceededError): + await limiter.is_key_within_model_budget(user_api_key_dict=_key(budget), model=checked_model) + else: + assert await limiter.is_key_within_model_budget(user_api_key_dict=_key(budget), model=checked_model) + + +@pytest.mark.asyncio +async def test_key_info_usage_reports_the_family_total(): + budget: Final = {"claude-opus*": ROOMY_CAP} + limiter: Final = _family_limiter() + + await _spend_on(limiter, model="claude-opus-4-8", model_max_budget=budget) + await _spend_on(limiter, model="claude-opus-5-5", model_max_budget=budget) + + usage: Final = await build_model_max_budget_usage( + entity_type=Litellm_EntityType.KEY, + entity_id=KEY_HASH, + model_max_budget=budget, + cache=limiter.dual_cache, + ) + assert usage["claude-opus*"]["current_spend"] == round(CHAT_COST * 2, 4) + + +@pytest.mark.parametrize( + ("pattern", "model", "governs"), + [ + ("claude-opus*", "claude-opus-4-8", True), + ("claude-opus*", "anthropic/claude-opus-4-8", True), + ("anthropic/claude-opus*", "anthropic/claude-opus-4-8", True), + ("anthropic/claude-opus*", "claude-opus-4-8", False), + ("claude-opus*", "my-claude-opus-4-8", False), + ("claude-opus*", "claude-sonnet-5", False), + ("gpt-4.1*", "gpt-401-mini", False), + ("*", "claude-sonnet-5", True), + ], +) +def test_glob_entries_match_the_whole_model_name_with_only_star_as_a_wildcard( + pattern: str, model: str, governs: bool +): + resolved: Final = resolve_model_budget( + model=model, model_max_budget={pattern: FAMILY_CAP}, model_access_groups=_model_access_groups({}) + ) + + assert (resolved is not None and resolved.budget_model == pattern) is governs + + +@pytest.mark.asyncio +async def test_replicas_without_redis_enforce_one_family_cap_together(): + budget: Final = {"claude-opus*": FAMILY_CAP} + table: Final = _SharedWindowsDouble() + replica_a: Final = _family_limiter(shared_windows=table) + replica_b: Final = _family_limiter(shared_windows=table) + + await _spend_on(replica_a, model="claude-opus-4-8", model_max_budget=budget) + + with pytest.raises(litellm.BudgetExceededError, match="model=claude-opus-5-5"): + await replica_b.is_key_within_model_budget(user_api_key_dict=_key(budget), model="claude-opus-5-5") + + +@pytest.mark.asyncio +async def test_replicas_sharing_redis_enforce_the_family_cap_without_the_database(): + budget: Final = {"claude-opus*": FAMILY_CAP} + redis: Final = _SharedRedisDouble() + table: Final = _SharedWindowsDouble() + replica_a: Final = _family_limiter( + dual_cache=DualCache(redis_cache=redis), # pyright: ignore[reportArgumentType] # duck-typed Redis double + shared_windows=table, + ) + replica_b: Final = _family_limiter( + dual_cache=DualCache(redis_cache=redis), # pyright: ignore[reportArgumentType] # duck-typed Redis double + shared_windows=table, + ) + + await _spend_on(replica_a, model="claude-opus-4-8", model_max_budget=budget) + await _drain_redis_pushes() + + with pytest.raises(litellm.BudgetExceededError, match="model=claude-opus-5-5"): + await replica_b.is_key_within_model_budget(user_api_key_dict=_key(budget), model="claude-opus-5-5") + assert dict(table.rows) == {} + + +@pytest.mark.asyncio +async def test_a_replica_charges_the_open_shared_window_instead_of_restarting_it(): + budget: Final = {"claude-opus*": {"budget_limit": CHAT_COST * 1.5, "time_period": "1d"}} + table: Final = _SharedWindowsDouble() + clock: Final = _Clock() + clock.advance(time.time()) + replica_a: Final = _family_limiter(shared_windows=table, clock=clock.now) + replica_b: Final = _family_limiter(shared_windows=table, clock=clock.now) + + await _spend_on(replica_a, model="claude-opus-4-8", model_max_budget=budget) + clock.advance(3600) + await _spend_on(replica_b, model="claude-opus-5-5", model_max_budget=budget) + + for replica in (replica_a, replica_b): + with pytest.raises(litellm.BudgetExceededError): + await replica.is_key_within_model_budget(user_api_key_dict=_key(budget), model="claude-opus-5-5") + + +@pytest.mark.asyncio +async def test_key_info_on_one_replica_reports_spend_from_every_replica(): + budget: Final = {"claude-opus*": ROOMY_CAP} + table: Final = _SharedWindowsDouble() + replica_a: Final = _family_limiter(shared_windows=table) + replica_b: Final = _family_limiter(shared_windows=table) + + await _spend_on(replica_a, model="claude-opus-4-8", model_max_budget=budget) + await _spend_on(replica_b, model="claude-opus-5-5", model_max_budget=budget) + + usage: Final = await build_model_max_budget_usage( + entity_type=Litellm_EntityType.KEY, + entity_id=KEY_HASH, + model_max_budget=budget, + cache=replica_a.dual_cache, + shared_windows=table, + ) + assert usage["claude-opus*"]["current_spend"] == round(CHAT_COST * 2, 4) + assert usage["claude-opus*"]["budget_reset_at"] is not None From 65c33671833cc944147a2b59c1f7e42263342c17 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 20:34:13 -0700 Subject: [PATCH 3/6] fix(proxy): open each shared model budget window once for every replica, and match patterns in linear time Without Redis, the first replica to charge an entry in a new window now opens it in the window spend table and the others join it, so no flush replaces spend another replica already added. Budget patterns match by splitting on `*` instead of a regular expression, a refused request names the entry that refused it, and the key form no longer offers a typed pattern another row already budgets. --- litellm/integrations/otel/model/spans.py | 1 + .../proxy/db/budget_window_spend_writer.py | 36 +++++ .../hooks/model_budget_shared_windows.py | 77 +++++++--- .../proxy/hooks/model_max_budget_limiter.py | 133 ++++++++++++------ .../db/test_budget_window_spend_writer.py | 24 ++++ .../hooks/test_model_budget_shared_windows.py | 54 ++++++- .../hooks/test_model_max_budget_limiter.py | 69 +++++++-- .../test_key_management_endpoints.py | 5 + tests/unit/test_internal_context.py | 1 + .../ModelMaxBudgetEditor.integration.test.tsx | 23 ++- .../key_team_helpers/ModelMaxBudgetEditor.tsx | 3 +- .../src/components/shared/SearchSelect.tsx | 5 +- 12 files changed, 355 insertions(+), 76 deletions(-) diff --git a/litellm/integrations/otel/model/spans.py b/litellm/integrations/otel/model/spans.py index d8be607150c..59519bf3428 100644 --- a/litellm/integrations/otel/model/spans.py +++ b/litellm/integrations/otel/model/spans.py @@ -434,6 +434,7 @@ _POSTGRES_OPERATION_BY_CALL_TYPE: Final[Mapping[str, PostgresOperation]] = Mappi "reset_budget_windows": PostgresOperation("select", None), "write_budget_windows": PostgresOperation("update", None), "roll_window_spend_row": PostgresOperation("update", None), + "open_window_spend_row": PostgresOperation("upsert", None), "seed_window_spend": PostgresOperation("select", None), "select_window_spend_rows": PostgresOperation("select", None), "commit_window_spend_updates": PostgresOperation("upsert", None), diff --git a/litellm/proxy/db/budget_window_spend_writer.py b/litellm/proxy/db/budget_window_spend_writer.py index 82c95cfe998..99b49331932 100644 --- a/litellm/proxy/db/budget_window_spend_writer.py +++ b/litellm/proxy/db/budget_window_spend_writer.py @@ -64,6 +64,16 @@ _ROLL_WINDOW_SPEND_SQL: Final = ( "AND window_start < ($4::timestamptz AT TIME ZONE 'UTC')" ) +_OPEN_WINDOW_SPEND_SQL: Final = ( + 'INSERT INTO "LiteLLM_BudgetWindowSpend" ' + "(entity_type, entity_id, window_duration, window_start, spend, created_at, updated_at) " + "VALUES ($1, $2, $3, ($4::timestamptz AT TIME ZONE 'UTC'), 0, " + "($4::timestamptz AT TIME ZONE 'UTC'), ($4::timestamptz AT TIME ZONE 'UTC')) " + "ON CONFLICT (entity_type, entity_id, window_duration) DO UPDATE SET " + "window_start = EXCLUDED.window_start, spend = 0, updated_at = EXCLUDED.updated_at " + "WHERE \"LiteLLM_BudgetWindowSpend\".window_start <= ($5::timestamptz AT TIME ZONE 'UTC')" +) + _SEED_FROM_SPEND_LOGS_KEY_SQL: Final = ( "SELECT COALESCE(SUM(spend), 0.0) AS total, " "COALESCE(SUM(spend) FILTER (WHERE \"startTime\" < ($3::timestamptz AT TIME ZONE 'UTC')), 0.0) AS before_batch " @@ -389,3 +399,29 @@ async def roll_window_spend_row( to_naive_utc(new_window_start), to_naive_utc(datetime.now(timezone.utc)), ) + + +async def open_window_spend_row( + prisma_client: "PrismaClient", + entity_type: str, + entity_id: str, + window_duration: str, + window_start: datetime, + replaces_window_start: datetime | None, +) -> None: + """Start a window at `window_start` unless another writer already started one. + + The row is created when it does not exist, and moved onto the new window only + while it still holds `replaces_window_start`, the ended window the caller saw. + A writer that lost the race leaves the winner's window in place, so reading + the row back gives every writer the same window start. + """ + async with db_span("open_window_spend_row", "LiteLLM_BudgetWindowSpend"): + await prisma_client.db.execute_raw( + _OPEN_WINDOW_SPEND_SQL, + entity_type, + entity_id, + window_duration, + to_naive_utc(window_start), + None if replaces_window_start is None else to_naive_utc(replaces_window_start), + ) diff --git a/litellm/proxy/hooks/model_budget_shared_windows.py b/litellm/proxy/hooks/model_budget_shared_windows.py index 344114edaa4..56729a33034 100644 --- a/litellm/proxy/hooks/model_budget_shared_windows.py +++ b/litellm/proxy/hooks/model_budget_shared_windows.py @@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Final, Protocol, TypeAlias from litellm._logging import verbose_proxy_logger from litellm.caching.in_memory_cache import InMemoryCache +from litellm.proxy.db.budget_window_spend_writer import open_window_spend_row from litellm.proxy.db.db_transaction_queue.window_spend_update_queue import ( WindowSpendTransaction, build_window_spend_transaction, @@ -37,6 +38,8 @@ class LocalTTLCache(Protocol): class SharedWindows(Protocol): async def read(self, spend_key: str, budget_duration: str) -> SharedWindow | None: ... + async def open(self, spend_key: str, budget_duration: str, now: float) -> float: ... + async def add(self, spend_key: str, budget_duration: str, started_at: float, spend: float) -> None: ... @@ -83,6 +86,40 @@ class DatabaseSharedWindows: self._readings.set_cache(key=spend_key, value=_Reading(window=window), ttl=self._read_ttl_seconds) return window + async def open(self, spend_key: str, budget_duration: str, now: float) -> float: + """When the window every replica shares started, opening one at `now` if none is open. + + Falls back to `now`, this replica's own window, when the database cannot answer. + """ + prisma_client: Final = self._prisma_client_source() + if prisma_client is None: + return now + try: + seen: Final = await _fetch_row( + prisma_client=prisma_client, spend_key=spend_key, budget_duration=budget_duration + ) + if seen is not None and seen.is_open(budget_duration=budget_duration, now=now): + return seen.started_at + await open_window_spend_row( + prisma_client=prisma_client, + entity_type=MODEL_BUDGET_WINDOW_ENTITY_TYPE, + entity_id=spend_key, + window_duration=budget_duration, + window_start=datetime.fromtimestamp(now, tz=timezone.utc), + replaces_window_start=None + if seen is None + else datetime.fromtimestamp(seen.started_at, tz=timezone.utc), + ) + opened: Final = await _fetch_row( + prisma_client=prisma_client, spend_key=spend_key, budget_duration=budget_duration + ) + except Exception: # noqa: BLE001 # an unreachable database leaves this replica on its own window + verbose_proxy_logger.exception("Unable to open the shared model budget window") + return now + if opened is None or not opened.is_open(budget_duration=budget_duration, now=now): + return now + return opened.started_at + async def add(self, spend_key: str, budget_duration: str, started_at: float, spend: float) -> None: if self._prisma_client_source() is None: return @@ -97,32 +134,36 @@ class DatabaseSharedWindows: ) ) except Exception: # noqa: BLE001 # spend tracking must never fail the success callback - verbose_proxy_logger.debug("Unable to queue shared model budget spend for %s", spend_key, exc_info=True) + verbose_proxy_logger.debug("Unable to queue shared model budget spend", exc_info=True) async def _read_row(self, spend_key: str, budget_duration: str) -> SharedWindow | None: prisma_client: Final = self._prisma_client_source() if prisma_client is None: return None try: - row: Final = await BudgetWindowSpendRepository(prisma_client).table.find_unique( - where={ - "entity_type_entity_id_window_duration": { - "entity_type": MODEL_BUDGET_WINDOW_ENTITY_TYPE, - "entity_id": spend_key, - "window_duration": budget_duration, - } - } - ) + return await _fetch_row(prisma_client=prisma_client, spend_key=spend_key, budget_duration=budget_duration) except Exception: # noqa: BLE001 # an unreadable row leaves enforcement on this replica's own counter - verbose_proxy_logger.exception("Unable to read shared model budget spend for %s", spend_key) + verbose_proxy_logger.exception("Unable to read shared model budget spend") return None - if row is None: - return None - window_start: Final = row.window_start - started_at: Final = ( - window_start if window_start.tzinfo is not None else window_start.replace(tzinfo=timezone.utc) - ).timestamp() - return SharedWindow(started_at=started_at, spend=float(row.spend or 0.0)) + + +async def _fetch_row(prisma_client: "PrismaClient", spend_key: str, budget_duration: str) -> SharedWindow | None: + row: Final = await BudgetWindowSpendRepository(prisma_client).table.find_unique( + where={ + "entity_type_entity_id_window_duration": { + "entity_type": MODEL_BUDGET_WINDOW_ENTITY_TYPE, + "entity_id": spend_key, + "window_duration": budget_duration, + } + } + ) + if row is None: + return None + window_start: Final = row.window_start + started_at: Final = ( + window_start if window_start.tzinfo is not None else window_start.replace(tzinfo=timezone.utc) + ).timestamp() + return SharedWindow(started_at=started_at, spend=float(row.spend or 0.0)) PROXY_SHARED_WINDOWS: Final = DatabaseSharedWindows() diff --git a/litellm/proxy/hooks/model_max_budget_limiter.py b/litellm/proxy/hooks/model_max_budget_limiter.py index d742e8148a7..1d7b7245d8a 100644 --- a/litellm/proxy/hooks/model_max_budget_limiter.py +++ b/litellm/proxy/hooks/model_max_budget_limiter.py @@ -1,7 +1,6 @@ import asyncio import functools import json -import re import time from collections.abc import Callable, Collection, Iterable, Iterator, Mapping, Sequence from dataclasses import dataclass @@ -195,13 +194,29 @@ def _governing_entry_names( def _glob_matches(pattern: str, names: Iterable[str]) -> bool: - regex: Final = _glob_regex(pattern) - return any(regex.fullmatch(name) for name in names) + return any(_glob_matches_name(pattern=pattern, name=name) for name in names) -@functools.lru_cache(maxsize=1024) -def _glob_regex(pattern: str) -> re.Pattern[str]: - return re.compile(re.escape(pattern).replace(re.escape(_GLOB_WILDCARD), ".*")) +def _glob_matches_name(pattern: str, name: str) -> bool: + """Whole-name match where `*` is the only wildcard, in linear time whatever the pattern holds. + + Taking each literal piece between two wildcards at its first occurrence is + always safe, since a later occurrence only leaves less room for the rest, so + the match never needs to backtrack the way a `.*` regex does. + """ + head, *middle, tail = pattern.split(_GLOB_WILDCARD) + middle_end: Final = len(name) - len(tail) + if middle_end < len(head) or not name.startswith(head) or not name.endswith(tail): + return False + return functools.reduce(_after_piece(name=name, end=middle_end), middle, len(head)) >= 0 + + +def _after_piece(name: str, end: int) -> Callable[[int, str], int]: + def step(position: int, piece: str) -> int: + found: Final = name.find(piece, position, end) if position >= 0 else -1 + return found + len(piece) if found >= 0 else -1 + + return step def _glob_precedence(pattern: str) -> tuple[tuple[int, int], str]: @@ -650,16 +665,26 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting): if max_budget is None or max_budget < 0: return True - current_spend: Final = await self._get_spend_for_model_budget( + local_spend: Final = await self._get_spend_for_model_budget( entity_type=entity_type, entity_id=entity_id, model=model, resolved=resolved, - max_budget=max_budget, + ) + current_spend: Final = ( + local_spend + if local_spend >= max_budget + else await self._with_shared_spend( + entity_type=entity_type, entity_id=entity_id, resolved=resolved, local_spend=local_spend + ) ) if current_spend >= max_budget: raise litellm.BudgetExceededError( - message=exceeded_message, + message=( + exceeded_message + if resolved.budget_model == model + else f"{exceeded_message}, budget_model={resolved.budget_model}" + ), current_cost=current_spend, max_budget=max_budget, entity_type=entity_type.value, @@ -673,16 +698,12 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting): entity_id: str | None, model: str, resolved: ResolvedModelBudget, - max_budget: float, ) -> float: """Spend charged to this budget in the current window, legacy counter included. A counter that was never written is zero spend, not unknown spend. The distinction only shows up at a zero-dollar cap, where skipping the comparison would let the strictest possible limit admit every request. - - Without Redis each replica's counter holds only its own share, so the - window every replica adds to in the database is the floor. """ spend_key: Final = model_budget_spend_cache_key( entity_type=entity_type, @@ -696,17 +717,30 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting): model=model, resolved=resolved, ) - counted: Final = _as_spend(await self._cached_spend(spend_key)) - local_spend: Final = ( - counted - if legacy_spend_key is None or legacy_spend_key == spend_key - else counted + _as_spend(await self._cached_spend(legacy_spend_key)) - ) - if local_spend >= max_budget or self.dual_cache.redis_cache is not None: + current_spend: Final = _as_spend(await self._cached_spend(spend_key)) + if legacy_spend_key is None or legacy_spend_key == spend_key: + return current_spend + return current_spend + _as_spend(await self._cached_spend(legacy_spend_key)) + + async def _with_shared_spend( + self, + entity_type: Litellm_EntityType, + entity_id: str | None, + resolved: ResolvedModelBudget, + local_spend: float, + ) -> float: + """Without Redis each replica's counter holds only its own share, so the window every replica adds to + in the database is the floor.""" + if self.dual_cache.redis_cache is not None: return local_spend shared: Final = await _open_shared_window( shared_windows=self._shared_windows, - spend_key=spend_key, + spend_key=model_budget_spend_cache_key( + entity_type=entity_type, + entity_id=entity_id, + budget_model=resolved.budget_model, + budget_duration=resolved.budget_config.budget_duration, + ), budget_duration=resolved.budget_config.budget_duration, now=self._clock(), ) @@ -878,6 +912,9 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting): budget_model=resolved.budget_model, budget_duration=budget_duration, ) + await self._join_shared_window( + spend_key=spend_key, start_time_key=start_time_key, budget_duration=budget_duration + ) await self._increment_spend_for_key( budget_config=resolved.budget_config, spend_key=spend_key, @@ -891,32 +928,35 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting): response_cost=response_cost, ) - async def _share_spend( - self, spend_key: str, start_time_key: str, budget_duration: str, response_cost: float - ) -> None: - """Without Redis, add this charge to the window every replica reads. + async def _join_shared_window(self, spend_key: str, start_time_key: str, budget_duration: str) -> None: + """Without Redis, start this replica's window where the database says every replica's window starts. - A replica joins the window that is already open rather than its own, so - all of them roll over together and none resets the others' spend. + Every replica then sends the same window start with its spend, so no flush reads as a newer window that + replaces the spend the others already added. """ if self.dual_cache.redis_cache is not None: return now: Final = self._clock() - shared: Final = await _open_shared_window( - shared_windows=self._shared_windows, + local: Final = self.dual_cache.in_memory_cache + if _open_local_window_start( + local=local, start_time_key=start_time_key, budget_duration=budget_duration, now=now + ): + return + _start_local_window( + local=local, spend_key=spend_key, + start_time_key=start_time_key, + started_at=await self._shared_windows.open(spend_key=spend_key, budget_duration=budget_duration, now=now), budget_duration=budget_duration, now=now, ) - if shared is not None: - _join_window( - local=self.dual_cache.in_memory_cache, - spend_key=spend_key, - start_time_key=start_time_key, - window=shared, - budget_duration=budget_duration, - now=now, - ) + + async def _share_spend( + self, spend_key: str, start_time_key: str, budget_duration: str, response_cost: float + ) -> None: + if self.dual_cache.redis_cache is not None: + return + now: Final = self._clock() await self._shared_windows.add( spend_key=spend_key, budget_duration=budget_duration, @@ -936,14 +976,17 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting): return polls == 1 -def _join_window( - local: LocalTTLCache, spend_key: str, start_time_key: str, window: SharedWindow, budget_duration: str, now: float +def _open_local_window_start(local: LocalTTLCache, start_time_key: str, budget_duration: str, now: float) -> bool: + started_at: Final = _as_float(local.get_cache(key=start_time_key)) + return started_at is not None and now - started_at <= budget_window_seconds(started_at, budget_duration) + + +def _start_local_window( + local: LocalTTLCache, spend_key: str, start_time_key: str, started_at: float, budget_duration: str, now: float ) -> None: - remaining: Final = window.started_at + budget_window_seconds(window.started_at, budget_duration) - now - local.set_cache(key=start_time_key, value=window.started_at, ttl=remaining) - counted: Final = local.get_cache(key=spend_key) - if counted is not None: - local.set_cache(key=spend_key, value=counted, ttl=remaining) + ttl: Final = started_at + budget_window_seconds(started_at, budget_duration) - now + local.set_cache(key=spend_key, value=0.0, ttl=ttl) + local.set_cache(key=start_time_key, value=started_at, ttl=ttl) PROXY_VirtualKeyModelMaxBudgetLimiter = _PROXY_VirtualKeyModelMaxBudgetLimiter diff --git a/tests/unit/proxy/db/test_budget_window_spend_writer.py b/tests/unit/proxy/db/test_budget_window_spend_writer.py index 6fc438feee8..27f1472c183 100644 --- a/tests/unit/proxy/db/test_budget_window_spend_writer.py +++ b/tests/unit/proxy/db/test_budget_window_spend_writer.py @@ -9,6 +9,7 @@ import pytest from litellm.proxy.db.budget_window_spend_writer import ( WindowSeedTotals, commit_window_spend_updates, + open_window_spend_row, roll_window_spend_row, spend_logs_seed_totals, ) @@ -405,6 +406,29 @@ async def test_roll_window_spend_row_is_conditional_on_the_stored_window_being_o assert params[:4] == ("team", "t1", "30d", datetime(2026, 8, 31)) +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("replaces_window_start", "replaced_param"), [(None, None), (WINDOW_A, datetime(2026, 8, 1))] +) +async def test_open_window_spend_row_moves_only_the_window_the_caller_saw_end(replaces_window_start, replaced_param): + db = _FakeDB() + + await open_window_spend_row( + prisma_client=_FakePrismaClient(db), + entity_type="model_max_budget", + entity_id="k1", + window_duration="1d", + window_start=WINDOW_B, + replaces_window_start=replaces_window_start, + ) + + ((query, params),) = db.execute_raw_calls + normalized = " ".join(query.split()) + assert "ON CONFLICT (entity_type, entity_id, window_duration) DO UPDATE" in normalized + assert normalized.endswith("WHERE \"LiteLLM_BudgetWindowSpend\".window_start <= ($5::timestamptz AT TIME ZONE 'UTC')") + assert params == ("model_max_budget", "k1", "1d", datetime(2026, 8, 31), replaced_param) + + @pytest.mark.asyncio async def test_seed_receives_the_batch_earliest_start_as_its_cutoff(): db = _FakeDB(existing_rows=[]) diff --git a/tests/unit/proxy/hooks/test_model_budget_shared_windows.py b/tests/unit/proxy/hooks/test_model_budget_shared_windows.py index 70957b1f74d..db334119b2a 100644 --- a/tests/unit/proxy/hooks/test_model_budget_shared_windows.py +++ b/tests/unit/proxy/hooks/test_model_budget_shared_windows.py @@ -1,4 +1,4 @@ -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone from types import SimpleNamespace from typing import Final @@ -24,6 +24,17 @@ class _Table: return self.row +class _Database: + def __init__(self, table: _Table) -> None: + self.litellm_budgetwindowspend = table + self.opened_with: tuple[object, ...] = () + + async def execute_raw(self, query: str, *params: object) -> int: + self.opened_with = params + self.litellm_budgetwindowspend.row = SimpleNamespace(window_start=params[3], spend=0.0) + return 1 + + class _Sink: def __init__(self) -> None: self.transactions: tuple[WindowSpendTransaction, ...] = () @@ -33,7 +44,7 @@ class _Sink: def _database_windows(table: _Table | None, sink: _Sink) -> DatabaseSharedWindows: - prisma: Final = None if table is None else SimpleNamespace(db=SimpleNamespace(litellm_budgetwindowspend=table)) + prisma: Final = None if table is None else SimpleNamespace(db=_Database(table)) return DatabaseSharedWindows( prisma_client_source=lambda: prisma, # pyright: ignore[reportArgumentType] # duck-typed prisma double sink=sink, @@ -91,3 +102,42 @@ async def test_nothing_is_queued_without_a_database(): assert sink.transactions == () assert await windows.read(spend_key=SPEND_KEY, budget_duration="1d") is None + + +@pytest.mark.asyncio +async def test_open_joins_the_window_another_replica_already_started(): + table: Final = _Table(row=SimpleNamespace(window_start=WINDOW_START.replace(tzinfo=None), spend=2.5)) + windows: Final = _database_windows(table, _Sink()) + + started_at: Final = await windows.open( + spend_key=SPEND_KEY, budget_duration="1d", now=WINDOW_START.timestamp() + 3600 + ) + + assert started_at == WINDOW_START.timestamp() + assert table.row.spend == 2.5 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stored_start", [None, WINDOW_START - timedelta(days=2)]) +async def test_open_starts_a_window_at_now_when_none_is_open(stored_start: datetime | None): + table: Final = _Table( + row=None if stored_start is None else SimpleNamespace(window_start=stored_start.replace(tzinfo=None), spend=9.0) + ) + windows: Final = _database_windows(table, _Sink()) + + started_at: Final = await windows.open(spend_key=SPEND_KEY, budget_duration="1d", now=WINDOW_START.timestamp()) + + assert started_at == WINDOW_START.timestamp() + assert await windows.read(spend_key=SPEND_KEY, budget_duration="1d") == SharedWindow( + started_at=WINDOW_START.timestamp(), spend=0.0 + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("table", [None, _Table(error=RuntimeError("database is down"))]) +async def test_open_falls_back_to_this_replicas_own_window_without_a_database(table: _Table | None): + windows: Final = _database_windows(table, _Sink()) + + assert await windows.open(spend_key=SPEND_KEY, budget_duration="1d", now=WINDOW_START.timestamp()) == ( + WINDOW_START.timestamp() + ) diff --git a/tests/unit/proxy/hooks/test_model_max_budget_limiter.py b/tests/unit/proxy/hooks/test_model_max_budget_limiter.py index 33fa7bc3daf..73328f0616f 100644 --- a/tests/unit/proxy/hooks/test_model_max_budget_limiter.py +++ b/tests/unit/proxy/hooks/test_model_max_budget_limiter.py @@ -1,4 +1,6 @@ import asyncio +import functools +import re import time from collections.abc import Callable, Mapping from types import MappingProxyType @@ -333,18 +335,37 @@ def _model_access_groups(groups_by_model: Mapping[str, frozenset[str]]) -> Calla class _SharedWindowsDouble: def __init__(self) -> None: self.rows: Mapping[str, SharedWindow] = MappingProxyType({}) + self.queued: tuple[tuple[str, float, float], ...] = () async def read(self, spend_key: str, budget_duration: str) -> SharedWindow | None: return self.rows.get(spend_key) - async def add(self, spend_key: str, budget_duration: str, started_at: float, spend: float) -> None: + async def open(self, spend_key: str, budget_duration: str, now: float) -> float: stored: Final = self.rows.get(spend_key) - row: Final = ( - SharedWindow(started_at=stored.started_at, spend=stored.spend + spend) - if stored is not None and stored.started_at >= started_at - else SharedWindow(started_at=started_at, spend=spend) - ) - self.rows = MappingProxyType({**self.rows, spend_key: row}) + if stored is not None and stored.is_open(budget_duration=budget_duration, now=now): + return stored.started_at + self.rows = MappingProxyType({**self.rows, spend_key: SharedWindow(started_at=now, spend=0.0)}) + return now + + async def add(self, spend_key: str, budget_duration: str, started_at: float, spend: float) -> None: + self.queued = (*self.queued, (spend_key, started_at, spend)) + + def flush(self) -> None: + self.rows = functools.reduce(_upsert_window_spend, self.queued, self.rows) + self.queued = () + + +def _upsert_window_spend( + rows: Mapping[str, SharedWindow], queued: tuple[str, float, float] +) -> Mapping[str, SharedWindow]: + spend_key, started_at, spend = queued + stored: Final = rows.get(spend_key) + row: Final = ( + SharedWindow(started_at=stored.started_at, spend=stored.spend + spend) + if stored is not None and stored.started_at >= started_at + else SharedWindow(started_at=started_at, spend=spend) + ) + return MappingProxyType({**rows, spend_key: row}) def _family_limiter( @@ -393,7 +414,9 @@ async def test_every_opus_version_draws_from_one_family_entry(family_entry: str) await _spend_on(limiter, model="claude-opus-4-8", model_max_budget=budget) - with pytest.raises(litellm.BudgetExceededError, match="model=claude-opus-5-5"): + with pytest.raises( + litellm.BudgetExceededError, match=rf"model=claude-opus-5-5, budget_model={re.escape(family_entry)}$" + ): await limiter.is_key_within_model_budget(user_api_key_dict=_key(budget), model="claude-opus-5-5") assert await limiter.is_key_within_model_budget(user_api_key_dict=_key(budget), model="claude-sonnet-5") @@ -488,6 +511,7 @@ async def test_replicas_without_redis_enforce_one_family_cap_together(): replica_b: Final = _family_limiter(shared_windows=table) await _spend_on(replica_a, model="claude-opus-4-8", model_max_budget=budget) + table.flush() with pytest.raises(litellm.BudgetExceededError, match="model=claude-opus-5-5"): await replica_b.is_key_within_model_budget(user_api_key_dict=_key(budget), model="claude-opus-5-5") @@ -509,6 +533,7 @@ async def test_replicas_sharing_redis_enforce_the_family_cap_without_the_databas await _spend_on(replica_a, model="claude-opus-4-8", model_max_budget=budget) await _drain_redis_pushes() + table.flush() with pytest.raises(litellm.BudgetExceededError, match="model=claude-opus-5-5"): await replica_b.is_key_within_model_budget(user_api_key_dict=_key(budget), model="claude-opus-5-5") @@ -525,8 +550,10 @@ async def test_a_replica_charges_the_open_shared_window_instead_of_restarting_it replica_b: Final = _family_limiter(shared_windows=table, clock=clock.now) await _spend_on(replica_a, model="claude-opus-4-8", model_max_budget=budget) + table.flush() clock.advance(3600) await _spend_on(replica_b, model="claude-opus-5-5", model_max_budget=budget) + table.flush() for replica in (replica_a, replica_b): with pytest.raises(litellm.BudgetExceededError): @@ -542,6 +569,7 @@ async def test_key_info_on_one_replica_reports_spend_from_every_replica(): await _spend_on(replica_a, model="claude-opus-4-8", model_max_budget=budget) await _spend_on(replica_b, model="claude-opus-5-5", model_max_budget=budget) + table.flush() usage: Final = await build_model_max_budget_usage( entity_type=Litellm_EntityType.KEY, @@ -552,3 +580,28 @@ async def test_key_info_on_one_replica_reports_spend_from_every_replica(): ) assert usage["claude-opus*"]["current_spend"] == round(CHAT_COST * 2, 4) assert usage["claude-opus*"]["budget_reset_at"] is not None + + +@pytest.mark.asyncio +@pytest.mark.parametrize("budget_model", ["claude-opus*", "claude-opus-4-8"]) +async def test_replicas_starting_a_window_moments_apart_keep_each_others_spend(budget_model: str): + budget: Final = {budget_model: ROOMY_CAP} + table: Final = _SharedWindowsDouble() + clock: Final = _Clock() + clock.advance(time.time()) + replica_a: Final = _family_limiter(shared_windows=table, clock=clock.now) + replica_b: Final = _family_limiter(shared_windows=table, clock=clock.now) + + await _spend_on(replica_a, model="claude-opus-4-8", model_max_budget=budget) + clock.advance(1) + await _spend_on(replica_b, model="claude-opus-4-8", model_max_budget=budget) + table.flush() + + usage: Final = await build_model_max_budget_usage( + entity_type=Litellm_EntityType.KEY, + entity_id=KEY_HASH, + model_max_budget=budget, + cache=DualCache(), + shared_windows=table, + ) + assert usage[budget_model]["current_spend"] == round(CHAT_COST * 2, 4) diff --git a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py index d6ee9b1d29e..fdc40f05146 100644 --- a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py @@ -16582,6 +16582,7 @@ async def test_info_key_fn_includes_model_max_budget_usage(monkeypatch): } mock_prisma_client = AsyncMock() + mock_prisma_client.db.litellm_budgetwindowspend.find_unique = AsyncMock(return_value=None) monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) mock_user_api_key_cache = AsyncMock() monkeypatch.setattr( @@ -16705,6 +16706,7 @@ async def test_info_key_fn_v2_includes_model_max_budget_usage(monkeypatch): model_max_budget = {"gpt-4o": {"budget_limit": 1.00, "time_period": "7d"}} mock_prisma_client = AsyncMock() + mock_prisma_client.db.litellm_budgetwindowspend.find_unique = AsyncMock(return_value=None) monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) mock_user_api_key_cache = AsyncMock() monkeypatch.setattr( @@ -16769,6 +16771,7 @@ async def test_info_key_fn_budget_table_fallback(monkeypatch): } mock_prisma_client = AsyncMock() + mock_prisma_client.db.litellm_budgetwindowspend.find_unique = AsyncMock(return_value=None) monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) mock_user_api_key_cache = AsyncMock() monkeypatch.setattr( @@ -16842,6 +16845,7 @@ async def test_info_key_fn_v2_budget_table_fallback(monkeypatch): } mock_prisma_client = AsyncMock() + mock_prisma_client.db.litellm_budgetwindowspend.find_unique = AsyncMock(return_value=None) monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) mock_user_api_key_cache = AsyncMock() monkeypatch.setattr( @@ -17217,6 +17221,7 @@ async def test_info_key_fn_reads_the_configured_budget_model_key(monkeypatch): } mock_prisma_client = AsyncMock() + mock_prisma_client.db.litellm_budgetwindowspend.find_unique = AsyncMock(return_value=None) monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) mock_user_api_key_cache = AsyncMock() monkeypatch.setattr( diff --git a/tests/unit/test_internal_context.py b/tests/unit/test_internal_context.py index 60a75c688c4..713d91e3016 100644 --- a/tests/unit/test_internal_context.py +++ b/tests/unit/test_internal_context.py @@ -74,6 +74,7 @@ _IN_MEMORY_ONLY_CALLERS: Final = frozenset( "litellm/proxy/container_endpoints/ownership.py", "litellm/proxy/discovery_endpoints/agent_skills_endpoints.py", "litellm/proxy/guardrails/guardrail_hooks/straiker/straiker.py", + "litellm/proxy/hooks/model_budget_shared_windows.py", "litellm/proxy/spend_tracking/key_metadata_recovery.py", "litellm/proxy/ui_crud_endpoints/latest_release_endpoints.py", "litellm/responses/litellm_completion_transformation/transformation.py", diff --git a/ui/litellm-dashboard/src/components/key_team_helpers/ModelMaxBudgetEditor.integration.test.tsx b/ui/litellm-dashboard/src/components/key_team_helpers/ModelMaxBudgetEditor.integration.test.tsx index c8fb108f66d..f7f0acd3254 100644 --- a/ui/litellm-dashboard/src/components/key_team_helpers/ModelMaxBudgetEditor.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/key_team_helpers/ModelMaxBudgetEditor.integration.test.tsx @@ -83,11 +83,32 @@ describe("ModelMaxBudgetEditor model families", () => { ); await user.click(addButton()); - const modelField = screen.getByPlaceholderText("Select a model or type a pattern"); + const modelField = screen.getByPlaceholderText("Select model"); await user.type(modelField, "claude-opus*"); await user.click(await screen.findByRole("option", { name: /claude-opus\*/ })); fireEvent.change(screen.getByPlaceholderText("Max spend ($)"), { target: { value: "10" } }); expect(onChange).toHaveBeenLastCalledWith({ "claude-opus*": { budget_limit: 10, time_period: "30d" } }); }); + + it("does not offer a typed pattern that another row already budgets", async () => { + const user = userEvent.setup(); + renderWithProviders( + , + ); + + await user.click(addButton()); + const [, newRowModelField] = screen.getAllByPlaceholderText("Select model"); + await user.type(newRowModelField, "claude-opus*"); + + expect(screen.queryByRole("option", { name: /claude-opus\*/ })).not.toBeInTheDocument(); + await user.clear(newRowModelField); + await user.type(newRowModelField, "claude-sonnet*"); + expect(await screen.findByRole("option", { name: /claude-sonnet\*/ })).toBeInTheDocument(); + }); }); diff --git a/ui/litellm-dashboard/src/components/key_team_helpers/ModelMaxBudgetEditor.tsx b/ui/litellm-dashboard/src/components/key_team_helpers/ModelMaxBudgetEditor.tsx index 0fc0f2116be..304d0a6ad1d 100644 --- a/ui/litellm-dashboard/src/components/key_team_helpers/ModelMaxBudgetEditor.tsx +++ b/ui/litellm-dashboard/src/components/key_team_helpers/ModelMaxBudgetEditor.tsx @@ -163,9 +163,10 @@ export function ModelMaxBudgetEditor({ options={modelOptions.map((model) => ({ label: model, value: model }))} value={entry.model} onValueChange={(model) => updateEntry(entry.id, { model })} - placeholder="Select a model or type a pattern" + placeholder="Select model" emptyText="No models found" customValueHint="Access group or model pattern" + reservedValues={entries.flatMap((other) => (other.id !== entry.id && other.model ? [other.model] : []))} disabled={!premiumUser} /> diff --git a/ui/litellm-dashboard/src/components/shared/SearchSelect.tsx b/ui/litellm-dashboard/src/components/shared/SearchSelect.tsx index 343194dc046..9b1203771d3 100644 --- a/ui/litellm-dashboard/src/components/shared/SearchSelect.tsx +++ b/ui/litellm-dashboard/src/components/shared/SearchSelect.tsx @@ -30,6 +30,7 @@ interface SearchSelectProps { inputId?: string; allowClear?: boolean; customValueHint?: string; + reservedValues?: readonly string[]; "aria-label"?: string; id?: string; } @@ -51,6 +52,7 @@ export function SearchSelect({ inputId, allowClear = true, customValueHint, + reservedValues = [], "aria-label": ariaLabel, id, }: SearchSelectProps) { @@ -60,8 +62,9 @@ export function SearchSelect({ const known = selected !== null && !options.some((option) => option.value === selected.value) ? [selected, ...options] : options; const typed = query.trim(); + const isNewValue = typed !== "" && !reservedValues.includes(typed) && !known.some((option) => option.value === typed); const items = - customValueHint !== undefined && typed !== "" && !known.some((option) => option.value === typed) + customValueHint !== undefined && isNewValue ? [...known, { label: typed, value: typed, sublabel: customValueHint }] : known; From afcfa6a35b6de46cc4323c1d6607663db3309661 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 21:03:20 -0700 Subject: [PATCH 4/6] fix(proxy): read the shared budget window from the writer, and keep concurrent first charges on one replica Opening a shared window now reads the row from the writer, since a lagging read replica could hand a replica its own start and let a flush replace the others' spend. A charge that waited on the open no longer resets a local window another charge on the same replica opened meanwhile. Window tests use fixed timestamps. --- .../hooks/model_budget_shared_windows.py | 14 ++++--- .../proxy/hooks/model_max_budget_limiter.py | 9 ++++- .../hooks/test_model_budget_shared_windows.py | 30 +++++++++++++- .../hooks/test_model_max_budget_limiter.py | 40 ++++++++++++++++--- 4 files changed, 79 insertions(+), 14 deletions(-) diff --git a/litellm/proxy/hooks/model_budget_shared_windows.py b/litellm/proxy/hooks/model_budget_shared_windows.py index 56729a33034..21f44ae6d6d 100644 --- a/litellm/proxy/hooks/model_budget_shared_windows.py +++ b/litellm/proxy/hooks/model_budget_shared_windows.py @@ -96,7 +96,7 @@ class DatabaseSharedWindows: return now try: seen: Final = await _fetch_row( - prisma_client=prisma_client, spend_key=spend_key, budget_duration=budget_duration + prisma_client=prisma_client, spend_key=spend_key, budget_duration=budget_duration, use_writer=True ) if seen is not None and seen.is_open(budget_duration=budget_duration, now=now): return seen.started_at @@ -111,7 +111,7 @@ class DatabaseSharedWindows: else datetime.fromtimestamp(seen.started_at, tz=timezone.utc), ) opened: Final = await _fetch_row( - prisma_client=prisma_client, spend_key=spend_key, budget_duration=budget_duration + prisma_client=prisma_client, spend_key=spend_key, budget_duration=budget_duration, use_writer=True ) except Exception: # noqa: BLE001 # an unreachable database leaves this replica on its own window verbose_proxy_logger.exception("Unable to open the shared model budget window") @@ -141,14 +141,18 @@ class DatabaseSharedWindows: if prisma_client is None: return None try: - return await _fetch_row(prisma_client=prisma_client, spend_key=spend_key, budget_duration=budget_duration) + return await _fetch_row( + prisma_client=prisma_client, spend_key=spend_key, budget_duration=budget_duration, use_writer=False + ) except Exception: # noqa: BLE001 # an unreadable row leaves enforcement on this replica's own counter verbose_proxy_logger.exception("Unable to read shared model budget spend") return None -async def _fetch_row(prisma_client: "PrismaClient", spend_key: str, budget_duration: str) -> SharedWindow | None: - row: Final = await BudgetWindowSpendRepository(prisma_client).table.find_unique( +async def _fetch_row( + prisma_client: "PrismaClient", spend_key: str, budget_duration: str, use_writer: bool +) -> SharedWindow | None: + row: Final = await BudgetWindowSpendRepository(prisma_client, use_writer=use_writer).table.find_unique( where={ "entity_type_entity_id_window_duration": { "entity_type": MODEL_BUDGET_WINDOW_ENTITY_TYPE, diff --git a/litellm/proxy/hooks/model_max_budget_limiter.py b/litellm/proxy/hooks/model_max_budget_limiter.py index 1d7b7245d8a..b8ac75aec3a 100644 --- a/litellm/proxy/hooks/model_max_budget_limiter.py +++ b/litellm/proxy/hooks/model_max_budget_limiter.py @@ -938,6 +938,13 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting): return now: Final = self._clock() local: Final = self.dual_cache.in_memory_cache + if _open_local_window_start( + local=local, start_time_key=start_time_key, budget_duration=budget_duration, now=now + ): + return + started_at: Final = await self._shared_windows.open( + spend_key=spend_key, budget_duration=budget_duration, now=now + ) if _open_local_window_start( local=local, start_time_key=start_time_key, budget_duration=budget_duration, now=now ): @@ -946,7 +953,7 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting): local=local, spend_key=spend_key, start_time_key=start_time_key, - started_at=await self._shared_windows.open(spend_key=spend_key, budget_duration=budget_duration, now=now), + started_at=started_at, budget_duration=budget_duration, now=now, ) diff --git a/tests/unit/proxy/hooks/test_model_budget_shared_windows.py b/tests/unit/proxy/hooks/test_model_budget_shared_windows.py index db334119b2a..8b103c33d32 100644 --- a/tests/unit/proxy/hooks/test_model_budget_shared_windows.py +++ b/tests/unit/proxy/hooks/test_model_budget_shared_windows.py @@ -43,8 +43,22 @@ class _Sink: self.transactions = (*self.transactions, transaction) -def _database_windows(table: _Table | None, sink: _Sink) -> DatabaseSharedWindows: - prisma: Final = None if table is None else SimpleNamespace(db=_Database(table)) +class _LaggingReplica: + def __init__(self, writer: _Database) -> None: + self.litellm_budgetwindowspend = _Table() + self._writer = writer + + async def execute_raw(self, query: str, *params: object) -> int: + return await self._writer.execute_raw(query, *params) + + +def _database_windows(table: _Table | None, sink: _Sink, lagging_replica: bool = False) -> DatabaseSharedWindows: + writer: Final = None if table is None else _Database(table) + prisma: Final = ( + None + if writer is None + else SimpleNamespace(db=_LaggingReplica(writer) if lagging_replica else writer, writer_db=writer) + ) return DatabaseSharedWindows( prisma_client_source=lambda: prisma, # pyright: ignore[reportArgumentType] # duck-typed prisma double sink=sink, @@ -117,6 +131,18 @@ async def test_open_joins_the_window_another_replica_already_started(): assert table.row.spend == 2.5 +@pytest.mark.asyncio +async def test_open_reads_the_window_from_the_writer_while_the_read_replica_lags(): + table: Final = _Table(row=SimpleNamespace(window_start=WINDOW_START.replace(tzinfo=None), spend=2.5)) + windows: Final = _database_windows(table, _Sink(), lagging_replica=True) + + started_at: Final = await windows.open( + spend_key=SPEND_KEY, budget_duration="1d", now=WINDOW_START.timestamp() + 3600 + ) + + assert started_at == WINDOW_START.timestamp() + + @pytest.mark.asyncio @pytest.mark.parametrize("stored_start", [None, WINDOW_START - timedelta(days=2)]) async def test_open_starts_a_window_at_now_when_none_is_open(stored_start: datetime | None): diff --git a/tests/unit/proxy/hooks/test_model_max_budget_limiter.py b/tests/unit/proxy/hooks/test_model_max_budget_limiter.py index 73328f0616f..cf925273163 100644 --- a/tests/unit/proxy/hooks/test_model_max_budget_limiter.py +++ b/tests/unit/proxy/hooks/test_model_max_budget_limiter.py @@ -3,6 +3,7 @@ import functools import re import time from collections.abc import Callable, Mapping +from datetime import datetime, timezone from types import MappingProxyType from typing import Final @@ -325,7 +326,11 @@ OPUS_GROUP_MEMBERS: Final = MappingProxyType( {"claude-opus-4-8": frozenset({OPUS_GROUP}), "claude-opus-5-5": frozenset({OPUS_GROUP})} ) FAMILY_CAP: Final = {"budget_limit": CHAT_COST, "time_period": "1d"} # mutable-ok: BudgetConfig validates dicts only -ROOMY_CAP: Final = {"budget_limit": CHAT_COST * 100, "time_period": "1d"} # mutable-ok: BudgetConfig validates dicts only +ROOMY_CAP: Final = { + "budget_limit": CHAT_COST * 100, + "time_period": "1d", +} # mutable-ok: BudgetConfig validates dicts only +WINDOW_EPOCH: Final = datetime(2026, 10, 20, 9, 30, tzinfo=timezone.utc).timestamp() def _model_access_groups(groups_by_model: Mapping[str, frozenset[str]]) -> Callable[[str], frozenset[str]]: @@ -341,6 +346,7 @@ class _SharedWindowsDouble: return self.rows.get(spend_key) async def open(self, spend_key: str, budget_duration: str, now: float) -> float: + await asyncio.sleep(0) stored: Final = self.rows.get(spend_key) if stored is not None and stored.is_open(budget_duration=budget_duration, now=now): return stored.started_at @@ -493,9 +499,7 @@ async def test_key_info_usage_reports_the_family_total(): ("*", "claude-sonnet-5", True), ], ) -def test_glob_entries_match_the_whole_model_name_with_only_star_as_a_wildcard( - pattern: str, model: str, governs: bool -): +def test_glob_entries_match_the_whole_model_name_with_only_star_as_a_wildcard(pattern: str, model: str, governs: bool): resolved: Final = resolve_model_budget( model=model, model_max_budget={pattern: FAMILY_CAP}, model_access_groups=_model_access_groups({}) ) @@ -545,7 +549,7 @@ async def test_a_replica_charges_the_open_shared_window_instead_of_restarting_it budget: Final = {"claude-opus*": {"budget_limit": CHAT_COST * 1.5, "time_period": "1d"}} table: Final = _SharedWindowsDouble() clock: Final = _Clock() - clock.advance(time.time()) + clock.advance(WINDOW_EPOCH) replica_a: Final = _family_limiter(shared_windows=table, clock=clock.now) replica_b: Final = _family_limiter(shared_windows=table, clock=clock.now) @@ -588,7 +592,7 @@ async def test_replicas_starting_a_window_moments_apart_keep_each_others_spend(b budget: Final = {budget_model: ROOMY_CAP} table: Final = _SharedWindowsDouble() clock: Final = _Clock() - clock.advance(time.time()) + clock.advance(WINDOW_EPOCH) replica_a: Final = _family_limiter(shared_windows=table, clock=clock.now) replica_b: Final = _family_limiter(shared_windows=table, clock=clock.now) @@ -602,6 +606,30 @@ async def test_replicas_starting_a_window_moments_apart_keep_each_others_spend(b entity_id=KEY_HASH, model_max_budget=budget, cache=DualCache(), + now=datetime.fromtimestamp(clock.now(), tz=timezone.utc), shared_windows=table, ) assert usage[budget_model]["current_spend"] == round(CHAT_COST * 2, 4) + + +@pytest.mark.asyncio +async def test_concurrent_first_charges_on_one_replica_both_count(): + budget: Final = {"claude-opus*": ROOMY_CAP} + clock: Final = _Clock() + clock.advance(WINDOW_EPOCH) + limiter: Final = _family_limiter(shared_windows=_SharedWindowsDouble(), clock=clock.now) + + await asyncio.gather( + _spend_on(limiter, model="claude-opus-4-8", model_max_budget=budget), + _spend_on(limiter, model="claude-opus-5-5", model_max_budget=budget), + ) + + usage: Final = await build_model_max_budget_usage( + entity_type=Litellm_EntityType.KEY, + entity_id=KEY_HASH, + model_max_budget=budget, + cache=limiter.dual_cache, + now=datetime.fromtimestamp(clock.now(), tz=timezone.utc), + shared_windows=_SharedWindowsDouble(), + ) + assert usage["claude-opus*"]["current_spend"] == round(CHAT_COST * 2, 4) From 053580ffce5288f49d60e16c10ab7c87d4ebaa93 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 22:13:44 -0700 Subject: [PATCH 5/6] fix(proxy): stop a non proxy admin from lifting a team model cap with a looser group or pattern entry Group and pattern entries now take precedence over other entries, so a team update by anyone but a proxy admin must keep each new or changed entry within every existing cap it can take precedence over. A pattern can match a provider-prefixed or wildcard-routed name, so any two patterns count as overlapping, and a model or group entry counts as outranking every pattern. --- .../proxy/hooks/model_max_budget_limiter.py | 53 ++++++++++ .../management_endpoints/team_endpoints.py | 77 ++++++++++++++- .../test_team_endpoints.py | 98 +++++++++++++++++++ 3 files changed, 223 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/hooks/model_max_budget_limiter.py b/litellm/proxy/hooks/model_max_budget_limiter.py index b8ac75aec3a..d6f5bb1c5bf 100644 --- a/litellm/proxy/hooks/model_max_budget_limiter.py +++ b/litellm/proxy/hooks/model_max_budget_limiter.py @@ -55,6 +55,8 @@ _GLOB_WILDCARD: Final = "*" ModelAccessGroups: TypeAlias = Callable[[str], Collection[str]] +AccessGroupMembers: TypeAlias = Callable[[str], Collection[str]] + _BUDGET_START_TIME_KEY_PREFIXES: Final = MappingProxyType( { Litellm_EntityType.KEY: "virtual_key_budget_start_time", @@ -153,6 +155,16 @@ def proxy_model_access_groups(model: str) -> frozenset[str]: return frozenset(llm_router.get_model_access_groups(model_name=model)) +def proxy_access_group_members(access_group: str) -> frozenset[str]: + try: + from litellm.proxy.proxy_server import llm_router + except ImportError: + return frozenset() + if llm_router is None: + return frozenset() + return frozenset(llm_router.get_model_access_groups(model_access_group=access_group).get(access_group, ())) + + def resolve_model_budget( model: str, model_max_budget: Mapping[str, object], @@ -223,6 +235,47 @@ def _glob_precedence(pattern: str) -> tuple[tuple[int, int], str]: return PatternUtils.calculate_pattern_specificity(pattern), pattern +def entries_outranked_by( + entry: str, + model_max_budget: Collection[str], + access_group_members: AccessGroupMembers = proxy_access_group_members, +) -> Iterator[str]: + """Entries that `entry` can take precedence over for some request model. + + A pattern can match a provider-prefixed or wildcard-routed name whose + candidates or access groups include any other entry, so patterns are taken + to overlap: a pattern outranks every less specific pattern, and a model or + group entry outranks every pattern. + """ + if _GLOB_WILDCARD in entry: + entry_precedence: Final = _glob_precedence(entry) + yield from ( + other + for other in model_max_budget + if _GLOB_WILDCARD in other and _glob_precedence(other) < entry_precedence + ) + return + yield from (other for other in model_max_budget if _GLOB_WILDCARD in other) + entry_members: Final = frozenset(access_group_members(entry)) + yield from ( + other + for other in model_max_budget + if _GLOB_WILDCARD not in other + and _ranks_ahead_of_group( + entry=entry, entry_members=entry_members, group=other, group_members=access_group_members(other) + ) + ) + + +def _ranks_ahead_of_group(entry: str, entry_members: frozenset[str], group: str, group_members: Iterable[str]) -> bool: + return any( + _GLOB_WILDCARD in member + or entry in _budget_model_candidates(member) + or (member in entry_members and entry < group) + for member in group_members + ) + + def _usable_entry(entry: str, raw_budget_config: object) -> ResolvedModelBudget | None: if (budget_config := _usable_budget_config(raw_budget_config)) is None: # An entry that will not validate cannot be keyed, so it cannot be diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index d9763f72741..c1b81a345d0 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -137,7 +137,10 @@ from litellm.proxy.common_utils.user_api_key_cache import ( from litellm.proxy.db.db_span import db_span from litellm.proxy.hooks.key_management_event_hooks import KeyManagementEventHooks from litellm.proxy.hooks.model_max_budget_limiter import ( + AccessGroupMembers, build_model_max_budget_usage, + entries_outranked_by, + proxy_access_group_members, resolve_model_budget, ) from litellm.proxy.management.teams.authz import TEAM_OR_ORG_ADMIN, TeamRole, is_team_admin, team_access_denied @@ -1427,6 +1430,7 @@ def _check_team_model_budget_update_authority( data: UpdateTeamRequest, user_api_key_dict: UserAPIKeyAuth, existing_model_max_budget: Mapping[str, object] | None, + access_group_members: AccessGroupMembers = proxy_access_group_members, ) -> None: """Like `_check_team_budget_update_authority`: only a proxy admin may raise, re-window or drop a per-model cap.""" if user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN: @@ -1454,11 +1458,7 @@ def _check_team_model_budget_update_authority( cap = governing.budget_config if cap.max_budget is None: continue - if ( - proposed.max_budget is None - or proposed.max_budget > cap.max_budget - or proposed.budget_duration != cap.budget_duration - ): + if _looser_model_cap(proposed=proposed, cap=cap): raise HTTPException( status_code=403, detail={ @@ -1470,6 +1470,73 @@ def _check_team_model_budget_update_authority( ) }, ) + for model_name, proposed in requested.items(): + _check_outranked_model_caps( + entry=model_name, + proposed=proposed, + existing_model_max_budget=existing_model_max_budget, + access_group_members=access_group_members, + ) + + +def _check_outranked_model_caps( + entry: str, + proposed: BudgetConfig, + existing_model_max_budget: Mapping[str, object], + access_group_members: AccessGroupMembers, +) -> None: + if _keeps_or_tightens(proposed=proposed, existing=_existing_model_cap(existing_model_max_budget.get(entry))): + return + outranked: Final = next( + _looser_than_outranked_caps(entry, proposed, existing_model_max_budget, access_group_members), None + ) + if outranked is None: + return + outranked_name, cap = outranked + raise HTTPException( + status_code=403, + detail={ + "error": ( + f"Only a proxy admin can set a team's model_max_budget entry {entry!r} looser than " + f"{outranked_name!r}, which it can take precedence over. Current max_budget={cap.max_budget} per " + f"{cap.budget_duration} (entry {outranked_name!r}), requested={proposed.max_budget} per " + f"{proposed.budget_duration}." + ) + }, + ) + + +def _looser_than_outranked_caps( + entry: str, + proposed: BudgetConfig, + existing_model_max_budget: Mapping[str, object], + access_group_members: AccessGroupMembers, +) -> Iterator[tuple[str, BudgetConfig]]: + outranked_caps: Final = ( + (outranked, _existing_model_cap(existing_model_max_budget[outranked])) + for outranked in entries_outranked_by( + entry=entry, model_max_budget=existing_model_max_budget, access_group_members=access_group_members + ) + ) + yield from ( + (outranked, cap) + for outranked, cap in outranked_caps + if cap is not None and cap.max_budget is not None and _looser_model_cap(proposed=proposed, cap=cap) + ) + + +def _keeps_or_tightens(proposed: BudgetConfig, existing: BudgetConfig | None) -> bool: + return existing is not None and ( + existing.max_budget is None or not _looser_model_cap(proposed=proposed, cap=existing) + ) + + +def _looser_model_cap(proposed: BudgetConfig, cap: BudgetConfig) -> bool: + return ( + proposed.max_budget is None + or (cap.max_budget is not None and proposed.max_budget > cap.max_budget) + or proposed.budget_duration != cap.budget_duration + ) def _should_auto_add_team_creator( diff --git a/tests/unit/proxy/management_endpoints/test_team_endpoints.py b/tests/unit/proxy/management_endpoints/test_team_endpoints.py index 37028558af3..2d728b506e9 100644 --- a/tests/unit/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_team_endpoints.py @@ -17119,6 +17119,104 @@ def test_team_admin_can_tighten_or_keep_team_model_caps(requested) -> None: ) +_TEAM_OPUS_FAMILY_CAP: Final = MappingProxyType({"claude-opus*": {"max_budget": 10.0, "budget_duration": "1d"}}) +_TEAM_OPUS_GROUP_CAP: Final = MappingProxyType({"z-opus-group": {"max_budget": 10.0, "budget_duration": "1d"}}) +_TEAM_ACCESS_GROUP_MEMBERS: Final = MappingProxyType( + { + "z-opus-group": ("claude-opus-4-8",), + "a-opus-group": ("claude-opus-4-8", "claude-opus-5-5"), + "zz-opus-group": ("claude-opus-4-8",), + "a-gemini-group": ("gemini-3-pro",), + "z-anthropic-group": ("anthropic/*",), + } +) + + +def _team_access_group_members(access_group: str) -> tuple[str, ...]: + return _TEAM_ACCESS_GROUP_MEMBERS.get(access_group, ()) + + +def _check_team_admin_model_caps(existing: Mapping[str, object], requested: Mapping[str, object]) -> None: + from litellm.proxy.management_endpoints.team_endpoints import _check_team_model_budget_update_authority + + _check_team_model_budget_update_authority( + data=UpdateTeamRequest(team_id="t1", model_max_budget=dict(requested)), + user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, user_id="team-admin"), + existing_model_max_budget=existing, + access_group_members=_team_access_group_members, + ) + + +@pytest.mark.parametrize( + ("existing", "added"), + [ + (_TEAM_OPUS_FAMILY_CAP, {"opus-group": {"max_budget": 1000.0, "budget_duration": "1d"}}), + (_TEAM_OPUS_FAMILY_CAP, {"opus-group": {"max_budget": 10.0, "budget_duration": "7d"}}), + (_TEAM_OPUS_FAMILY_CAP, {"*laude-opus-4-8": {"max_budget": 1000.0, "budget_duration": "1d"}}), + (_TEAM_OPUS_GROUP_CAP, {"a-opus-group": {"max_budget": 1000.0, "budget_duration": "1d"}}), + (_TEAM_OPUS_GROUP_CAP, {"claude-opus-4-8": {"max_budget": 1000.0, "budget_duration": "1d"}}), + ( + {"z-anthropic-group": {"max_budget": 10.0, "budget_duration": "1d"}}, + {"claude-opus-4-8": {"max_budget": 1000.0, "budget_duration": "1d"}}, + ), + ], + ids=[ + "group_over_pattern", + "group_rewindows_pattern", + "more_specific_pattern", + "earlier_group_sharing_a_model", + "model_inside_group", + "model_under_wildcard_group", + ], +) +def test_team_admin_cannot_add_an_entry_looser_than_a_cap_it_outranks( + existing: Mapping[str, object], added: Mapping[str, object] +) -> None: + with pytest.raises(HTTPException) as exc: + _check_team_admin_model_caps(existing=existing, requested={**existing, **added}) + assert exc.value.status_code == 403 + assert "proxy admin" in exc.value.detail["error"] + assert repr(next(iter(existing))) in exc.value.detail["error"] + + +@pytest.mark.parametrize( + ("existing", "requested"), + [ + (_TEAM_OPUS_FAMILY_CAP, {**_TEAM_OPUS_FAMILY_CAP, "opus-group": {"max_budget": 5.0, "budget_duration": "1d"}}), + (_TEAM_OPUS_FAMILY_CAP, {**_TEAM_OPUS_FAMILY_CAP, "claude*": {"max_budget": 1000.0, "budget_duration": "1d"}}), + (_TEAM_OPUS_GROUP_CAP, {**_TEAM_OPUS_GROUP_CAP, "a-gemini-group": {"max_budget": 1000.0, "budget_duration": "1d"}}), + (_TEAM_OPUS_GROUP_CAP, {**_TEAM_OPUS_GROUP_CAP, "zz-opus-group": {"max_budget": 1000.0, "budget_duration": "1d"}}), + (_TEAM_OPUS_GROUP_CAP, {**_TEAM_OPUS_GROUP_CAP, "gpt-4o": {"max_budget": 1000.0, "budget_duration": "1d"}}), + ( + {**_TEAM_OPUS_FAMILY_CAP, "opus-group": {"max_budget": 100.0, "budget_duration": "1d"}}, + {**_TEAM_OPUS_FAMILY_CAP, "opus-group": {"max_budget": 100.0, "budget_duration": "1d"}}, + ), + ( + {**_TEAM_OPUS_FAMILY_CAP, "opus-group": {"max_budget": 100.0, "budget_duration": "1d"}}, + {**_TEAM_OPUS_FAMILY_CAP, "opus-group": {"max_budget": 50.0, "budget_duration": "1d"}}, + ), + ( + {**_TEAM_OPUS_FAMILY_CAP, "opus-group": {"rpm_limit": 10}}, + {**_TEAM_OPUS_FAMILY_CAP, "opus-group": {"rpm_limit": 20}}, + ), + ], + ids=[ + "tighter_group", + "less_specific_pattern", + "group_without_shared_models", + "later_group_sharing_a_model", + "model_outside_group", + "unchanged_looser_entry", + "tightened_looser_entry", + "uncapped_entry_edited", + ], +) +def test_team_admin_can_add_entries_that_never_lift_an_outranked_cap( + existing: Mapping[str, object], requested: Mapping[str, object] +) -> None: + assert _check_team_admin_model_caps(existing=existing, requested=requested) is None + + def test_team_model_cap_authority_skips_omitted_field_malformed_rows_and_proxy_admins() -> None: from litellm.proxy.management_endpoints.team_endpoints import _check_team_model_budget_update_authority From 3d78d57662dc6fc1dba53194e68d8831ae8cb3a3 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 21 Oct 2026 01:45:06 -0700 Subject: [PATCH 6/6] fix(proxy): narrow model budget pattern overlap, keep team caps under key patterns, and accept only typed patterns in the UI --- .../proxy/hooks/model_max_budget_limiter.py | 190 ++++++++++++------ .../management_endpoints/team_endpoints.py | 17 +- .../hooks/test_model_budget_shared_windows.py | 24 +++ .../hooks/test_model_max_budget_limiter.py | 86 ++++++++ .../test_team_endpoints.py | 53 +++-- .../ModelMaxBudgetEditor.integration.test.tsx | 18 ++ .../key_team_helpers/ModelMaxBudgetEditor.tsx | 4 +- .../shared/SearchSelect.integration.test.tsx | 7 +- .../src/components/shared/SearchSelect.tsx | 13 +- 9 files changed, 325 insertions(+), 87 deletions(-) diff --git a/litellm/proxy/hooks/model_max_budget_limiter.py b/litellm/proxy/hooks/model_max_budget_limiter.py index d6f5bb1c5bf..0531a079dea 100644 --- a/litellm/proxy/hooks/model_max_budget_limiter.py +++ b/litellm/proxy/hooks/model_max_budget_limiter.py @@ -1,12 +1,18 @@ import asyncio import functools +import itertools import json import time from collections.abc import Callable, Collection, Iterable, Iterator, Mapping, Sequence from dataclasses import dataclass from datetime import datetime, timezone from types import MappingProxyType -from typing import Final, TypeAlias, cast # noqa: TID251 # untyped callback payload requires explicit narrowing +from typing import ( + TYPE_CHECKING, + Final, + TypeAlias, + cast, # noqa: TID251 # untyped callback payload requires explicit narrowing +) import litellm from litellm._internal_context import with_service_target @@ -25,9 +31,11 @@ from litellm.proxy.hooks.model_budget_shared_windows import ( ) from litellm.router_strategy.budget_limiter import RouterBudgetLimiting, budget_window_seconds from litellm.router_utils.batch_utils import batch_charged_once_marker_key, batch_id_to_charge_once -from litellm.router_utils.pattern_match_deployments import PatternUtils from litellm.types.llms.openai import AllMessageValues -from litellm.types.utils import BudgetConfig, StandardLoggingPayload +from litellm.types.utils import BudgetConfig, LlmProviders, StandardLoggingPayload + +if TYPE_CHECKING: + from litellm.router import Router VIRTUAL_KEY_SPEND_CACHE_KEY_PREFIX: Final = "virtual_key_spend" END_USER_SPEND_CACHE_KEY_PREFIX: Final = "end_user_model_spend" @@ -47,6 +55,8 @@ _SPEND_CACHE_KEY_PREFIXES: Final = MappingProxyType( _LEGACY_REQUEST_MODEL_SCOPES: Final = frozenset({Litellm_EntityType.KEY, Litellm_EntityType.END_USER}) +_PER_REPLICA_SCOPES: Final = frozenset({Litellm_EntityType.END_USER}) + _PROCESS_STARTED_AT: Final = time.monotonic() _LATEST_RESET_AT: Final = datetime(9999, 12, 31, tzinfo=timezone.utc).timestamp() @@ -55,7 +65,7 @@ _GLOB_WILDCARD: Final = "*" ModelAccessGroups: TypeAlias = Callable[[str], Collection[str]] -AccessGroupMembers: TypeAlias = Callable[[str], Collection[str]] +_PROVIDER_PREFIXES: Final = tuple(f"{provider.value}/" for provider in LlmProviders) _BUDGET_START_TIME_KEY_PREFIXES: Final = MappingProxyType( { @@ -82,6 +92,15 @@ class ResolvedModelBudget: budget_config: BudgetConfig +@dataclass(frozen=True, slots=True) +class RoutedModels: + served_names: frozenset[str] + group_members: Mapping[str, frozenset[str]] + + def members(self, access_group: str) -> frozenset[str]: + return self.group_members.get(access_group, frozenset()) + + def team_member_budget_entity_id(user_id: str, team_id: str) -> str: return f"{len(user_id)}:{user_id}:{team_id}" @@ -145,24 +164,48 @@ def model_budget_start_time_cache_key( return f"{_BUDGET_START_TIME_KEY_PREFIXES[entity_type]}:{entity_id}:{budget_model}:{budget_duration}" +def model_access_groups_of(router: "Router", model: str) -> frozenset[str]: + """Access groups serving `model`, leaving out any named after a model the router serves. + + An entry with that name already budgets that model, so reading it as the group too would quietly widen it. + """ + return frozenset( + group for group in router.get_model_access_groups(model_name=model) if group not in router.model_names + ) + + +def routed_models_of(router: "Router") -> RoutedModels: + served_names: Final = frozenset(router.model_names) + return RoutedModels( + served_names=served_names, + group_members=MappingProxyType( + { + group: frozenset(members) + for group, members in router.get_model_access_groups().items() + if group not in served_names + } + ), + ) + + def proxy_model_access_groups(model: str) -> frozenset[str]: + router: Final = _proxy_router() + return frozenset() if router is None else model_access_groups_of(router=router, model=model) + + +def proxy_routed_models() -> RoutedModels: + router: Final = _proxy_router() + if router is None: + return RoutedModels(served_names=frozenset(), group_members=MappingProxyType({})) + return routed_models_of(router) + + +def _proxy_router() -> "Router | None": try: from litellm.proxy.proxy_server import llm_router except ImportError: - return frozenset() - if llm_router is None: - return frozenset() - return frozenset(llm_router.get_model_access_groups(model_name=model)) - - -def proxy_access_group_members(access_group: str) -> frozenset[str]: - try: - from litellm.proxy.proxy_server import llm_router - except ImportError: - return frozenset() - if llm_router is None: - return frozenset() - return frozenset(llm_router.get_model_access_groups(model_access_group=access_group).get(access_group, ())) + return None + return llm_router def resolve_model_budget( @@ -231,21 +274,21 @@ def _after_piece(name: str, end: int) -> Callable[[int, str], int]: return step -def _glob_precedence(pattern: str) -> tuple[tuple[int, int], str]: - return PatternUtils.calculate_pattern_specificity(pattern), pattern +def _glob_precedence(pattern: str) -> tuple[int, int, str]: + """More literal characters first, then fewer wildcards, so padding a pattern with `*` never makes it win.""" + wildcards: Final = pattern.count(_GLOB_WILDCARD) + return len(pattern) - wildcards, -wildcards, pattern -def entries_outranked_by( - entry: str, - model_max_budget: Collection[str], - access_group_members: AccessGroupMembers = proxy_access_group_members, -) -> Iterator[str]: - """Entries that `entry` can take precedence over for some request model. +def entries_outranked_by(entry: str, model_max_budget: Collection[str], routed_models: RoutedModels) -> Iterator[str]: + """Entries that `entry` can take precedence over for some model a request can name. - A pattern can match a provider-prefixed or wildcard-routed name whose - candidates or access groups include any other entry, so patterns are taken - to overlap: a pattern outranks every less specific pattern, and a model or - group entry outranks every pattern. + Two patterns can both match a name the router has not served yet, so a + pattern outranks every less specific pattern. A model or group entry + outranks a pattern that matches one of the names it governs: the model, + the model under any provider prefix or wildcard route, a served name + ending in it, or a member of the group, and every pattern when a member is + itself a wildcard route. """ if _GLOB_WILDCARD in entry: entry_precedence: Final = _glob_precedence(entry) @@ -255,18 +298,41 @@ def entries_outranked_by( if _GLOB_WILDCARD in other and _glob_precedence(other) < entry_precedence ) return - yield from (other for other in model_max_budget if _GLOB_WILDCARD in other) - entry_members: Final = frozenset(access_group_members(entry)) + entry_members: Final = routed_models.members(entry) + covers_wildcard_route: Final = any(_GLOB_WILDCARD in member for member in entry_members) + governed_candidates: Final = frozenset( + itertools.chain.from_iterable( + _budget_model_candidates(name) + for name in _names_governed_by(entry=entry, entry_members=entry_members, routed_models=routed_models) + ) + ) + yield from ( + other + for other in model_max_budget + if _GLOB_WILDCARD in other and (covers_wildcard_route or _glob_matches(other, governed_candidates)) + ) yield from ( other for other in model_max_budget if _GLOB_WILDCARD not in other and _ranks_ahead_of_group( - entry=entry, entry_members=entry_members, group=other, group_members=access_group_members(other) + entry=entry, entry_members=entry_members, group=other, group_members=routed_models.members(other) ) ) +def _names_governed_by(entry: str, entry_members: frozenset[str], routed_models: RoutedModels) -> Iterator[str]: + yield entry + yield from (f"{prefix}{entry}" for prefix in _PROVIDER_PREFIXES) + yield from ( + name + for name in routed_models.served_names + if _GLOB_WILDCARD not in name and entry in _budget_model_candidates(name) + ) + yield from (name.replace(_GLOB_WILDCARD, entry) for name in routed_models.served_names if _GLOB_WILDCARD in name) + yield from (member for member in entry_members if _GLOB_WILDCARD not in member) + + def _ranks_ahead_of_group(entry: str, entry_members: frozenset[str], group: str, group_members: Iterable[str]) -> bool: return any( _GLOB_WILDCARD in member @@ -295,16 +361,21 @@ def _usable_entry(entry: str, raw_budget_config: object) -> ResolvedModelBudget return ResolvedModelBudget(budget_model=entry, budget_config=budget_config) -def team_model_budget_applies( - model: str, - key_model_max_budget: Mapping[str, object] | None, - model_access_groups: ModelAccessGroups = proxy_model_access_groups, -) -> bool: - """A key entry that spend-gates `model` overrides the team cap: it is then gated on and billed to the key alone.""" +def team_model_budget_applies(model: str, key_model_max_budget: Mapping[str, object] | None) -> bool: + """A key entry named after `model` that spend-gates it overrides the team cap: it is then gated on and billed to + the key alone. A key pattern or group entry is gated alongside the team cap, never instead of it.""" if not key_model_max_budget: return True resolved: Final = resolve_model_budget( - model=model, model_max_budget=key_model_max_budget, model_access_groups=model_access_groups + model=model, + model_max_budget=MappingProxyType( + { + candidate: key_model_max_budget[candidate] + for candidate in _budget_model_candidates(model) + if candidate in key_model_max_budget + } + ), + model_access_groups=lambda _: frozenset(), ) return resolved is None or not _spend_gated(resolved.budget_config) @@ -667,9 +738,7 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting): Raises: BudgetExceededError: If the team has exceeded the model budget """ - if not team_model_budget_applies( - model=model, key_model_max_budget=key_model_max_budget, model_access_groups=self._model_access_groups - ): + if not team_model_budget_applies(model=model, key_model_max_budget=key_model_max_budget): return True return await self._is_entity_within_model_budget( entity_type=Litellm_EntityType.TEAM, @@ -735,7 +804,7 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting): raise litellm.BudgetExceededError( message=( exceeded_message - if resolved.budget_model == model + if resolved.budget_model in _budget_model_candidates(model) else f"{exceeded_message}, budget_model={resolved.budget_model}" ), current_cost=current_spend, @@ -784,7 +853,7 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting): ) -> float: """Without Redis each replica's counter holds only its own share, so the window every replica adds to in the database is the floor.""" - if self.dual_cache.redis_cache is not None: + if not self._shares_windows_in_database(entity_type): return local_spend shared: Final = await _open_shared_window( shared_windows=self._shared_windows, @@ -878,7 +947,6 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting): key_model_max_budget=( key_model_max_budget if isinstance(key_model_max_budget, Mapping) else None ), - model_access_groups=self._model_access_groups, ) else None ), @@ -965,21 +1033,29 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting): budget_model=resolved.budget_model, budget_duration=budget_duration, ) - await self._join_shared_window( - spend_key=spend_key, start_time_key=start_time_key, budget_duration=budget_duration - ) + shares_window: Final = self._shares_windows_in_database(entity_type) + if shares_window: + await self._join_shared_window( + spend_key=spend_key, start_time_key=start_time_key, budget_duration=budget_duration + ) await self._increment_spend_for_key( budget_config=resolved.budget_config, spend_key=spend_key, start_time_key=start_time_key, response_cost=response_cost, ) - await self._share_spend( - spend_key=spend_key, - start_time_key=start_time_key, - budget_duration=budget_duration, - response_cost=response_cost, - ) + if shares_window: + await self._share_spend( + spend_key=spend_key, + start_time_key=start_time_key, + budget_duration=budget_duration, + response_cost=response_cost, + ) + + def _shares_windows_in_database(self, entity_type: Litellm_EntityType) -> bool: + """End-user ids are whatever the client sends, so their windows stay on each replica rather than adding a + database row per id.""" + return self.dual_cache.redis_cache is None and entity_type not in _PER_REPLICA_SCOPES async def _join_shared_window(self, spend_key: str, start_time_key: str, budget_duration: str) -> None: """Without Redis, start this replica's window where the database says every replica's window starts. @@ -987,8 +1063,6 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting): Every replica then sends the same window start with its spend, so no flush reads as a newer window that replaces the spend the others already added. """ - if self.dual_cache.redis_cache is not None: - return now: Final = self._clock() local: Final = self.dual_cache.in_memory_cache if _open_local_window_start( @@ -1014,8 +1088,6 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting): async def _share_spend( self, spend_key: str, start_time_key: str, budget_duration: str, response_cost: float ) -> None: - if self.dual_cache.redis_cache is not None: - return now: Final = self._clock() await self._shared_windows.add( spend_key=spend_key, diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index c1b81a345d0..89c0d5e7b56 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -137,10 +137,10 @@ from litellm.proxy.common_utils.user_api_key_cache import ( from litellm.proxy.db.db_span import db_span from litellm.proxy.hooks.key_management_event_hooks import KeyManagementEventHooks from litellm.proxy.hooks.model_max_budget_limiter import ( - AccessGroupMembers, + RoutedModels, build_model_max_budget_usage, entries_outranked_by, - proxy_access_group_members, + proxy_routed_models, resolve_model_budget, ) from litellm.proxy.management.teams.authz import TEAM_OR_ORG_ADMIN, TeamRole, is_team_admin, team_access_denied @@ -1430,7 +1430,7 @@ def _check_team_model_budget_update_authority( data: UpdateTeamRequest, user_api_key_dict: UserAPIKeyAuth, existing_model_max_budget: Mapping[str, object] | None, - access_group_members: AccessGroupMembers = proxy_access_group_members, + routed_models_source: Callable[[], RoutedModels] = proxy_routed_models, ) -> None: """Like `_check_team_budget_update_authority`: only a proxy admin may raise, re-window or drop a per-model cap.""" if user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN: @@ -1470,12 +1470,13 @@ def _check_team_model_budget_update_authority( ) }, ) + routed_models: Final = routed_models_source() for model_name, proposed in requested.items(): _check_outranked_model_caps( entry=model_name, proposed=proposed, existing_model_max_budget=existing_model_max_budget, - access_group_members=access_group_members, + routed_models=routed_models, ) @@ -1483,12 +1484,12 @@ def _check_outranked_model_caps( entry: str, proposed: BudgetConfig, existing_model_max_budget: Mapping[str, object], - access_group_members: AccessGroupMembers, + routed_models: RoutedModels, ) -> None: if _keeps_or_tightens(proposed=proposed, existing=_existing_model_cap(existing_model_max_budget.get(entry))): return outranked: Final = next( - _looser_than_outranked_caps(entry, proposed, existing_model_max_budget, access_group_members), None + _looser_than_outranked_caps(entry, proposed, existing_model_max_budget, routed_models), None ) if outranked is None: return @@ -1510,12 +1511,12 @@ def _looser_than_outranked_caps( entry: str, proposed: BudgetConfig, existing_model_max_budget: Mapping[str, object], - access_group_members: AccessGroupMembers, + routed_models: RoutedModels, ) -> Iterator[tuple[str, BudgetConfig]]: outranked_caps: Final = ( (outranked, _existing_model_cap(existing_model_max_budget[outranked])) for outranked in entries_outranked_by( - entry=entry, model_max_budget=existing_model_max_budget, access_group_members=access_group_members + entry=entry, model_max_budget=existing_model_max_budget, routed_models=routed_models ) ) yield from ( diff --git a/tests/unit/proxy/hooks/test_model_budget_shared_windows.py b/tests/unit/proxy/hooks/test_model_budget_shared_windows.py index 8b103c33d32..c294d667315 100644 --- a/tests/unit/proxy/hooks/test_model_budget_shared_windows.py +++ b/tests/unit/proxy/hooks/test_model_budget_shared_windows.py @@ -35,6 +35,16 @@ class _Database: return 1 +class _OutracedDatabase: + def __init__(self, table: _Table, winner: object) -> None: + self.litellm_budgetwindowspend = table + self._winner = winner + + async def execute_raw(self, query: str, *params: object) -> int: + self.litellm_budgetwindowspend.row = self._winner + return 0 + + class _Sink: def __init__(self) -> None: self.transactions: tuple[WindowSpendTransaction, ...] = () @@ -143,6 +153,20 @@ async def test_open_reads_the_window_from_the_writer_while_the_read_replica_lags assert started_at == WINDOW_START.timestamp() +@pytest.mark.asyncio +async def test_open_joins_the_window_a_racing_replica_opened_first(): + winner: Final = SimpleNamespace(window_start=WINDOW_START.replace(tzinfo=None), spend=2.5) + database: Final = _OutracedDatabase(_Table(), winner) + windows: Final = DatabaseSharedWindows( + prisma_client_source=lambda: SimpleNamespace(db=database, writer_db=database), # pyright: ignore[reportArgumentType] # duck-typed prisma double + sink=_Sink(), + ) + + started_at: Final = await windows.open(spend_key=SPEND_KEY, budget_duration="1d", now=WINDOW_START.timestamp() + 60) + + assert started_at == WINDOW_START.timestamp() + + @pytest.mark.asyncio @pytest.mark.parametrize("stored_start", [None, WINDOW_START - timedelta(days=2)]) async def test_open_starts_a_window_at_now_when_none_is_open(stored_start: datetime | None): diff --git a/tests/unit/proxy/hooks/test_model_max_budget_limiter.py b/tests/unit/proxy/hooks/test_model_max_budget_limiter.py index cf925273163..e97be8abbb0 100644 --- a/tests/unit/proxy/hooks/test_model_max_budget_limiter.py +++ b/tests/unit/proxy/hooks/test_model_max_budget_limiter.py @@ -16,10 +16,14 @@ from litellm.proxy.hooks.model_budget_shared_windows import SharedWindow, Shared from litellm.proxy.hooks.model_max_budget_limiter import ( PROXY_VirtualKeyModelMaxBudgetLimiter, build_model_max_budget_usage, + model_access_groups_of, model_budget_spend_cache_key, resolve_model_budget, + routed_models_of, team_member_budget_entity_id, + team_model_budget_applies, ) +from litellm.router import Router from litellm.types.caching import RedisPipelineIncrementOperation from litellm.types.utils import LiteLLMBatch, Usage @@ -453,6 +457,7 @@ async def test_a_model_added_to_the_access_group_later_is_covered_with_no_key_ch ({"claude-opus-4*": FAMILY_CAP, "claude-opus*": ROOMY_CAP}, "claude-opus-4-7", True), ({"claude-opus-4*": FAMILY_CAP, "claude-opus*": ROOMY_CAP}, "claude-opus-5-5", False), ({"claude-opus-4*": {"budget_limit": CHAT_COST}, "claude-opus*": FAMILY_CAP}, "claude-opus-5-5", True), + ({"claude-opus-4*": FAMILY_CAP, "c" + "*" * 15: ROOMY_CAP}, "claude-opus-4-7", True), ], ) async def test_the_most_specific_entry_takes_the_spend_and_gates_the_request( @@ -469,6 +474,87 @@ async def test_the_most_specific_entry_takes_the_spend_and_gates_the_request( assert await limiter.is_key_within_model_budget(user_api_key_dict=_key(budget), model=checked_model) +@pytest.mark.asyncio +async def test_an_exact_entry_refusal_reads_as_before_for_a_provider_prefixed_model(): + budget: Final = {"claude-opus-4-8": FAMILY_CAP} + limiter: Final = _family_limiter() + + await _spend_on(limiter, model="claude-opus-4-8", model_max_budget=budget) + + with pytest.raises(litellm.BudgetExceededError, match=r"model=anthropic/claude-opus-4-8$"): + await limiter.is_key_within_model_budget(user_api_key_dict=_key(budget), model="anthropic/claude-opus-4-8") + + +@pytest.mark.parametrize( + ("key_entry", "team_cap_applies"), + [("claude-opus-4-8", False), ("anthropic/claude-opus-4-8", True), ("claude-opus*", True), (OPUS_GROUP, True)], +) +def test_only_a_key_entry_named_after_the_model_replaces_the_team_cap(key_entry: str, team_cap_applies: bool): + assert team_model_budget_applies(model="claude-opus-4-8", key_model_max_budget={key_entry: ROOMY_CAP}) is ( + team_cap_applies + ) + + +@pytest.mark.asyncio +async def test_end_user_windows_stay_on_the_replica_without_redis(): + budget: Final = {"claude-opus*": FAMILY_CAP} + table: Final = _SharedWindowsDouble() + limiter: Final = _family_limiter(shared_windows=table) + + await _log_success( + limiter, + { + "call_type": "acompletion", + "standard_logging_object": { + "call_type": "acompletion", + "response_cost": CHAT_COST, + "model": "anthropic/claude-opus-4-8", + "model_group": "claude-opus-4-8", + "end_user": "end-user-1", + "metadata": {}, + }, + "litellm_params": {"metadata": {"user_api_key_end_user_model_max_budget": budget}}, + }, + ) + table.flush() + + assert dict(table.rows) == {} + with pytest.raises(litellm.BudgetExceededError, match="model=claude-opus-5-5"): + await limiter.is_end_user_within_model_budget( + end_user_id="end-user-1", end_user_model_max_budget=budget, model="claude-opus-5-5" + ) + + +def test_an_access_group_named_after_a_served_model_never_widens_that_models_entry(): + router: Final = Router( + model_list=[ + { + "model_name": "gpt-4o", + "litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-test"}, + "model_info": {"access_groups": ["gpt-4o", "openai-models"]}, + }, + { + "model_name": "gpt-4o-mini", + "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-test"}, + "model_info": {"access_groups": ["gpt-4o", "openai-models"]}, + }, + ] + ) + + def governing_entry(model_max_budget: Mapping[str, object]) -> str | None: + resolved: Final = resolve_model_budget( + model="gpt-4o-mini", + model_max_budget=model_max_budget, + model_access_groups=lambda model: model_access_groups_of(router=router, model=model), + ) + return None if resolved is None else resolved.budget_model + + assert governing_entry({"gpt-4o": FAMILY_CAP}) is None + assert governing_entry({"openai-models": FAMILY_CAP}) == "openai-models" + assert routed_models_of(router).members("gpt-4o") == frozenset() + assert routed_models_of(router).members("openai-models") == frozenset({"gpt-4o", "gpt-4o-mini"}) + + @pytest.mark.asyncio async def test_key_info_usage_reports_the_family_total(): budget: Final = {"claude-opus*": ROOMY_CAP} diff --git a/tests/unit/proxy/management_endpoints/test_team_endpoints.py b/tests/unit/proxy/management_endpoints/test_team_endpoints.py index 2d728b506e9..7fbb639648e 100644 --- a/tests/unit/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_team_endpoints.py @@ -44,6 +44,7 @@ from litellm.proxy._types import ( UpdateTeamRequest, UserAPIKeyAuth, # Import UserAPIKeyAuth ) +from litellm.proxy.hooks.model_max_budget_limiter import RoutedModels from litellm.proxy.management.teams.authz import TeamAccess from litellm.proxy.management_endpoints.team_admin_field_permissions import ( PROXY_CONTROLLED_TEAM_METADATA_KEYS, @@ -17121,21 +17122,23 @@ def test_team_admin_can_tighten_or_keep_team_model_caps(requested) -> None: _TEAM_OPUS_FAMILY_CAP: Final = MappingProxyType({"claude-opus*": {"max_budget": 10.0, "budget_duration": "1d"}}) _TEAM_OPUS_GROUP_CAP: Final = MappingProxyType({"z-opus-group": {"max_budget": 10.0, "budget_duration": "1d"}}) -_TEAM_ACCESS_GROUP_MEMBERS: Final = MappingProxyType( - { - "z-opus-group": ("claude-opus-4-8",), - "a-opus-group": ("claude-opus-4-8", "claude-opus-5-5"), - "zz-opus-group": ("claude-opus-4-8",), - "a-gemini-group": ("gemini-3-pro",), - "z-anthropic-group": ("anthropic/*",), - } +_TEAM_ROUTED_MODELS: Final = RoutedModels( + served_names=frozenset( + {"claude-opus-4-8", "claude-opus-5-5", "gemini-3-pro", "gpt-4o", "my-team/gpt-4o", "team-route/*"} + ), + group_members=MappingProxyType( + { + "opus-group": frozenset({"claude-opus-4-8"}), + "z-opus-group": frozenset({"claude-opus-4-8"}), + "a-opus-group": frozenset({"claude-opus-4-8", "claude-opus-5-5"}), + "zz-opus-group": frozenset({"claude-opus-4-8"}), + "a-gemini-group": frozenset({"gemini-3-pro"}), + "z-anthropic-group": frozenset({"anthropic/*"}), + } + ), ) -def _team_access_group_members(access_group: str) -> tuple[str, ...]: - return _TEAM_ACCESS_GROUP_MEMBERS.get(access_group, ()) - - def _check_team_admin_model_caps(existing: Mapping[str, object], requested: Mapping[str, object]) -> None: from litellm.proxy.management_endpoints.team_endpoints import _check_team_model_budget_update_authority @@ -17143,7 +17146,7 @@ def _check_team_admin_model_caps(existing: Mapping[str, object], requested: Mapp data=UpdateTeamRequest(team_id="t1", model_max_budget=dict(requested)), user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, user_id="team-admin"), existing_model_max_budget=existing, - access_group_members=_team_access_group_members, + routed_models_source=lambda: _TEAM_ROUTED_MODELS, ) @@ -17159,6 +17162,19 @@ def _check_team_admin_model_caps(existing: Mapping[str, object], requested: Mapp {"z-anthropic-group": {"max_budget": 10.0, "budget_duration": "1d"}}, {"claude-opus-4-8": {"max_budget": 1000.0, "budget_duration": "1d"}}, ), + (_TEAM_OPUS_FAMILY_CAP, {"claude-opus-4-8": {"max_budget": 1000.0, "budget_duration": "1d"}}), + ( + {"anthropic/*": {"max_budget": 10.0, "budget_duration": "1d"}}, + {"claude-opus-4-8": {"max_budget": 1000.0, "budget_duration": "1d"}}, + ), + ( + {"my-team/*": {"max_budget": 10.0, "budget_duration": "1d"}}, + {"gpt-4o": {"max_budget": 1000.0, "budget_duration": "1d"}}, + ), + ( + {"team-route/*": {"max_budget": 10.0, "budget_duration": "1d"}}, + {"gpt-4o": {"max_budget": 1000.0, "budget_duration": "1d"}}, + ), ], ids=[ "group_over_pattern", @@ -17167,6 +17183,10 @@ def _check_team_admin_model_caps(existing: Mapping[str, object], requested: Mapp "earlier_group_sharing_a_model", "model_inside_group", "model_under_wildcard_group", + "model_matching_pattern", + "model_under_provider_pattern", + "model_under_served_prefix", + "model_under_wildcard_route", ], ) def test_team_admin_cannot_add_an_entry_looser_than_a_cap_it_outranks( @@ -17187,6 +17207,11 @@ def test_team_admin_cannot_add_an_entry_looser_than_a_cap_it_outranks( (_TEAM_OPUS_GROUP_CAP, {**_TEAM_OPUS_GROUP_CAP, "a-gemini-group": {"max_budget": 1000.0, "budget_duration": "1d"}}), (_TEAM_OPUS_GROUP_CAP, {**_TEAM_OPUS_GROUP_CAP, "zz-opus-group": {"max_budget": 1000.0, "budget_duration": "1d"}}), (_TEAM_OPUS_GROUP_CAP, {**_TEAM_OPUS_GROUP_CAP, "gpt-4o": {"max_budget": 1000.0, "budget_duration": "1d"}}), + (_TEAM_OPUS_FAMILY_CAP, {**_TEAM_OPUS_FAMILY_CAP, "gpt-4o": {"max_budget": 1000.0, "budget_duration": "1d"}}), + ( + _TEAM_OPUS_FAMILY_CAP, + {**_TEAM_OPUS_FAMILY_CAP, "a-gemini-group": {"max_budget": 1000.0, "budget_duration": "1d"}}, + ), ( {**_TEAM_OPUS_FAMILY_CAP, "opus-group": {"max_budget": 100.0, "budget_duration": "1d"}}, {**_TEAM_OPUS_FAMILY_CAP, "opus-group": {"max_budget": 100.0, "budget_duration": "1d"}}, @@ -17206,6 +17231,8 @@ def test_team_admin_cannot_add_an_entry_looser_than_a_cap_it_outranks( "group_without_shared_models", "later_group_sharing_a_model", "model_outside_group", + "model_outside_pattern", + "group_outside_pattern", "unchanged_looser_entry", "tightened_looser_entry", "uncapped_entry_edited", diff --git a/ui/litellm-dashboard/src/components/key_team_helpers/ModelMaxBudgetEditor.integration.test.tsx b/ui/litellm-dashboard/src/components/key_team_helpers/ModelMaxBudgetEditor.integration.test.tsx index f7f0acd3254..955b911ca97 100644 --- a/ui/litellm-dashboard/src/components/key_team_helpers/ModelMaxBudgetEditor.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/key_team_helpers/ModelMaxBudgetEditor.integration.test.tsx @@ -111,4 +111,22 @@ describe("ModelMaxBudgetEditor model families", () => { await user.type(newRowModelField, "claude-sonnet*"); expect(await screen.findByRole("option", { name: /claude-sonnet\*/ })).toBeInTheDocument(); }); + + it("does not offer a typed name that is neither listed nor a pattern", async () => { + const user = userEvent.setup(); + renderWithProviders( + , + ); + + await user.click(addButton()); + await user.type(screen.getByPlaceholderText("Select model"), "claude-opsu-4-8"); + + expect(screen.queryByRole("option", { name: /claude-opsu-4-8/ })).not.toBeInTheDocument(); + expect(screen.getByText("No models found")).toBeInTheDocument(); + }); }); diff --git a/ui/litellm-dashboard/src/components/key_team_helpers/ModelMaxBudgetEditor.tsx b/ui/litellm-dashboard/src/components/key_team_helpers/ModelMaxBudgetEditor.tsx index 304d0a6ad1d..1e321a59e02 100644 --- a/ui/litellm-dashboard/src/components/key_team_helpers/ModelMaxBudgetEditor.tsx +++ b/ui/litellm-dashboard/src/components/key_team_helpers/ModelMaxBudgetEditor.tsx @@ -50,6 +50,8 @@ export const MODEL_BUDGET_PERIOD_OPTIONS = [ const DEFAULT_PERIOD = "30d"; +const MODEL_PATTERN_VALUE = { hint: "Model pattern", accepts: (typed: string) => typed.includes("*") }; + export const entriesToModelMaxBudget = (entries: readonly ModelBudgetEntry[]): ModelMaxBudget => Object.fromEntries( entries @@ -165,7 +167,7 @@ export function ModelMaxBudgetEditor({ onValueChange={(model) => updateEntry(entry.id, { model })} placeholder="Select model" emptyText="No models found" - customValueHint="Access group or model pattern" + customValue={MODEL_PATTERN_VALUE} reservedValues={entries.flatMap((other) => (other.id !== entry.id && other.model ? [other.model] : []))} disabled={!premiumUser} /> diff --git a/ui/litellm-dashboard/src/components/shared/SearchSelect.integration.test.tsx b/ui/litellm-dashboard/src/components/shared/SearchSelect.integration.test.tsx index ce5b734941a..ecb8dd161f3 100644 --- a/ui/litellm-dashboard/src/components/shared/SearchSelect.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/shared/SearchSelect.integration.test.tsx @@ -63,16 +63,19 @@ describe("SearchSelect", () => { expect(onValueChange).toHaveBeenLastCalledWith("team-3"); }); - it("offers the typed text as a value only when the caller accepts custom values", async () => { + it("offers the typed text as a value only when the caller accepts it", async () => { const user = userEvent.setup(); const onValueChange = vi.fn(); + const customValue = { hint: "Custom value", accepts: (typed: string) => typed.endsWith("*") }; const { rerender } = render(); const input = screen.getByRole("combobox"); await user.click(input); fireEvent.change(input, { target: { value: "team-*" } }); expect(screen.queryByRole("option", { name: /team-\*/ })).not.toBeInTheDocument(); - rerender(); + rerender(); + fireEvent.change(input, { target: { value: "team-4" } }); + expect(screen.queryByRole("option", { name: /team-4/ })).not.toBeInTheDocument(); fireEvent.change(input, { target: { value: "team-*" } }); await user.click(await screen.findByRole("option", { name: /team-\*/ })); expect(onValueChange).toHaveBeenLastCalledWith("team-*"); diff --git a/ui/litellm-dashboard/src/components/shared/SearchSelect.tsx b/ui/litellm-dashboard/src/components/shared/SearchSelect.tsx index 9b1203771d3..e139720a2ed 100644 --- a/ui/litellm-dashboard/src/components/shared/SearchSelect.tsx +++ b/ui/litellm-dashboard/src/components/shared/SearchSelect.tsx @@ -19,6 +19,11 @@ export interface SearchSelectOption { icon?: React.ReactNode; } +interface CustomValue { + hint: string; + accepts: (typed: string) => boolean; +} + interface SearchSelectProps { options: SearchSelectOption[]; value?: string | null; @@ -29,7 +34,7 @@ interface SearchSelectProps { className?: string; inputId?: string; allowClear?: boolean; - customValueHint?: string; + customValue?: CustomValue; reservedValues?: readonly string[]; "aria-label"?: string; id?: string; @@ -51,7 +56,7 @@ export function SearchSelect({ className, inputId, allowClear = true, - customValueHint, + customValue, reservedValues = [], "aria-label": ariaLabel, id, @@ -64,8 +69,8 @@ export function SearchSelect({ const typed = query.trim(); const isNewValue = typed !== "" && !reservedValues.includes(typed) && !known.some((option) => option.value === typed); const items = - customValueHint !== undefined && isNewValue - ? [...known, { label: typed, value: typed, sublabel: customValueHint }] + customValue !== undefined && isNewValue && customValue.accepts(typed) + ? [...known, { label: typed, value: typed, sublabel: customValue.hint }] : known; return (