From 3d7347d22725380581b9160ba814fc79876a296d Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 20 Oct 2026 18:21:59 -0700
Subject: [PATCH 1/6] feat(ui): let the per-model budget field take a model
pattern or access group
---
.../ModelMaxBudgetEditor.integration.test.tsx | 26 ++++++++++++++++++-
.../key_team_helpers/ModelMaxBudgetEditor.tsx | 5 ++--
.../shared/SearchSelect.integration.test.tsx | 15 +++++++++++
.../src/components/shared/SearchSelect.tsx | 13 ++++++++--
4 files changed, 54 insertions(+), 5 deletions(-)
diff --git a/ui/litellm-dashboard/src/components/key_team_helpers/ModelMaxBudgetEditor.integration.test.tsx b/ui/litellm-dashboard/src/components/key_team_helpers/ModelMaxBudgetEditor.integration.test.tsx
index 33f09092cca..c8fb108f66d 100644
--- a/ui/litellm-dashboard/src/components/key_team_helpers/ModelMaxBudgetEditor.integration.test.tsx
+++ b/ui/litellm-dashboard/src/components/key_team_helpers/ModelMaxBudgetEditor.integration.test.tsx
@@ -1,5 +1,6 @@
+import userEvent from "@testing-library/user-event";
import { describe, expect, it, vi } from "vitest";
-import { renderWithProviders, screen } from "../../../tests/test-utils";
+import { fireEvent, renderWithProviders, screen } from "../../../tests/test-utils";
import { MODEL_MAX_BUDGET_PREMIUM_HINT, ModelMaxBudgetEditor, type ModelMaxBudget } from "./ModelMaxBudgetEditor";
const STORED: ModelMaxBudget = { "gpt-4o": { budget_limit: 5, time_period: "30d" } };
@@ -67,3 +68,26 @@ describe("ModelMaxBudgetEditor with an enterprise license", () => {
expect(addButton()).toBeEnabled();
});
});
+
+describe("ModelMaxBudgetEditor model families", () => {
+ it("stores a typed pattern that matches no listed model as one budget entry", async () => {
+ const user = userEvent.setup();
+ const onChange = vi.fn();
+ renderWithProviders(
+ ,
+ );
+
+ await user.click(addButton());
+ const modelField = screen.getByPlaceholderText("Select a model or type a pattern");
+ await user.type(modelField, "claude-opus*");
+ await user.click(await screen.findByRole("option", { name: /claude-opus\*/ }));
+ fireEvent.change(screen.getByPlaceholderText("Max spend ($)"), { target: { value: "10" } });
+
+ expect(onChange).toHaveBeenLastCalledWith({ "claude-opus*": { budget_limit: 10, time_period: "30d" } });
+ });
+});
diff --git a/ui/litellm-dashboard/src/components/key_team_helpers/ModelMaxBudgetEditor.tsx b/ui/litellm-dashboard/src/components/key_team_helpers/ModelMaxBudgetEditor.tsx
index a6e72068463..0fc0f2116be 100644
--- a/ui/litellm-dashboard/src/components/key_team_helpers/ModelMaxBudgetEditor.tsx
+++ b/ui/litellm-dashboard/src/components/key_team_helpers/ModelMaxBudgetEditor.tsx
@@ -121,7 +121,7 @@ export function ModelMaxBudgetEditor({
const blurb = (
{premiumUser
- ? "Cap spend per model over its own window. A budget set on the bare model name also covers the provider-prefixed spelling of that model."
+ ? "Cap spend per model over its own window. A budget set on the bare model name also covers the provider-prefixed spelling of that model. Type an access group or a pattern like claude-opus* to share one budget across every model it covers."
: MODEL_MAX_BUDGET_PREMIUM_HINT}
);
@@ -163,8 +163,9 @@ export function ModelMaxBudgetEditor({
options={modelOptions.map((model) => ({ label: model, value: model }))}
value={entry.model}
onValueChange={(model) => updateEntry(entry.id, { model })}
- placeholder="Select model"
+ placeholder="Select a model or type a pattern"
emptyText="No models found"
+ customValueHint="Access group or model pattern"
disabled={!premiumUser}
/>
diff --git a/ui/litellm-dashboard/src/components/shared/SearchSelect.integration.test.tsx b/ui/litellm-dashboard/src/components/shared/SearchSelect.integration.test.tsx
index 9f320049b09..ce5b734941a 100644
--- a/ui/litellm-dashboard/src/components/shared/SearchSelect.integration.test.tsx
+++ b/ui/litellm-dashboard/src/components/shared/SearchSelect.integration.test.tsx
@@ -63,6 +63,21 @@ describe("SearchSelect", () => {
expect(onValueChange).toHaveBeenLastCalledWith("team-3");
});
+ it("offers the typed text as a value only when the caller accepts custom values", async () => {
+ const user = userEvent.setup();
+ const onValueChange = vi.fn();
+ const { rerender } = render();
+ const input = screen.getByRole("combobox");
+ await user.click(input);
+ fireEvent.change(input, { target: { value: "team-*" } });
+ expect(screen.queryByRole("option", { name: /team-\*/ })).not.toBeInTheDocument();
+
+ rerender();
+ fireEvent.change(input, { target: { value: "team-*" } });
+ await user.click(await screen.findByRole("option", { name: /team-\*/ }));
+ expect(onValueChange).toHaveBeenLastCalledWith("team-*");
+ });
+
it("filters the options client-side as you type", async () => {
const user = userEvent.setup();
render();
diff --git a/ui/litellm-dashboard/src/components/shared/SearchSelect.tsx b/ui/litellm-dashboard/src/components/shared/SearchSelect.tsx
index 3649114826d..343194dc046 100644
--- a/ui/litellm-dashboard/src/components/shared/SearchSelect.tsx
+++ b/ui/litellm-dashboard/src/components/shared/SearchSelect.tsx
@@ -1,6 +1,6 @@
"use client";
-import React from "react";
+import React, { useState } from "react";
import {
Combobox,
ComboboxContent,
@@ -29,6 +29,7 @@ interface SearchSelectProps {
className?: string;
inputId?: string;
allowClear?: boolean;
+ customValueHint?: string;
"aria-label"?: string;
id?: string;
}
@@ -49,13 +50,20 @@ export function SearchSelect({
className,
inputId,
allowClear = true,
+ customValueHint,
"aria-label": ariaLabel,
id,
}: SearchSelectProps) {
+ const [query, setQuery] = useState("");
const selected =
value == null || value === "" ? null : options.find((option) => option.value === value) ?? { label: value, value };
- const items =
+ const known =
selected !== null && !options.some((option) => option.value === selected.value) ? [selected, ...options] : options;
+ const typed = query.trim();
+ const items =
+ customValueHint !== undefined && typed !== "" && !known.some((option) => option.value === typed)
+ ? [...known, { label: typed, value: typed, sublabel: customValueHint }]
+ : known;
return (
a.value === b.value}
itemToStringLabel={(item: SearchSelectOption) => item.label}
filter={matchesQuery}
+ onInputValueChange={setQuery}
disabled={disabled}
>
Date: Tue, 20 Oct 2026 19:08:13 -0700
Subject: [PATCH 2/6] feat(proxy): let model_max_budget entries name a model
family or access group, shared across replicas
---
.../hooks/model_budget_shared_windows.py | 128 +++++++
.../proxy/hooks/model_max_budget_limiter.py | 313 +++++++++++++++---
.../hooks/test_model_budget_shared_windows.py | 93 ++++++
.../hooks/test_model_max_budget_limiter.py | 257 +++++++++++++-
4 files changed, 729 insertions(+), 62 deletions(-)
create mode 100644 litellm/proxy/hooks/model_budget_shared_windows.py
create mode 100644 tests/unit/proxy/hooks/test_model_budget_shared_windows.py
diff --git a/litellm/proxy/hooks/model_budget_shared_windows.py b/litellm/proxy/hooks/model_budget_shared_windows.py
new file mode 100644
index 00000000000..344114edaa4
--- /dev/null
+++ b/litellm/proxy/hooks/model_budget_shared_windows.py
@@ -0,0 +1,128 @@
+from collections.abc import Awaitable, Callable
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from typing import TYPE_CHECKING, Final, Protocol, TypeAlias
+
+from litellm._logging import verbose_proxy_logger
+from litellm.caching.in_memory_cache import InMemoryCache
+from litellm.proxy.db.db_transaction_queue.window_spend_update_queue import (
+ WindowSpendTransaction,
+ build_window_spend_transaction,
+)
+from litellm.repositories.table_repositories import BudgetWindowSpendRepository
+from litellm.router_strategy.budget_limiter import budget_window_seconds
+
+if TYPE_CHECKING:
+ from litellm.proxy.utils import PrismaClient
+
+MODEL_BUDGET_WINDOW_ENTITY_TYPE: Final = "model_max_budget"
+_READ_TTL_SECONDS: Final = 5
+
+
+@dataclass(frozen=True, slots=True)
+class SharedWindow:
+ started_at: float
+ spend: float
+
+ def is_open(self, budget_duration: str, now: float) -> bool:
+ return now < self.started_at + budget_window_seconds(self.started_at, budget_duration)
+
+
+class LocalTTLCache(Protocol):
+ def get_cache(self, key: str) -> object: ...
+
+ def set_cache(self, key: str, value: object, *, ttl: float) -> None: ...
+
+
+class SharedWindows(Protocol):
+ async def read(self, spend_key: str, budget_duration: str) -> SharedWindow | None: ...
+
+ async def add(self, spend_key: str, budget_duration: str, started_at: float, spend: float) -> None: ...
+
+
+PrismaClientSource: TypeAlias = Callable[[], "PrismaClient | None"]
+WindowSpendSink: TypeAlias = Callable[[WindowSpendTransaction], Awaitable[None]]
+
+
+def _proxy_prisma_client() -> "PrismaClient | None":
+ try:
+ from litellm.proxy.proxy_server import prisma_client
+ except ImportError:
+ return None
+ return prisma_client
+
+
+async def _queue_on_proxy_spend_writer(transaction: WindowSpendTransaction) -> None:
+ from litellm.proxy.proxy_server import proxy_logging_obj
+
+ await proxy_logging_obj.db_spend_update_writer.window_spend_update_queue.add_update(transaction)
+
+
+@dataclass(frozen=True, slots=True)
+class _Reading:
+ window: SharedWindow | None
+
+
+class DatabaseSharedWindows:
+ def __init__(
+ self,
+ prisma_client_source: PrismaClientSource = _proxy_prisma_client,
+ sink: WindowSpendSink = _queue_on_proxy_spend_writer,
+ read_ttl_seconds: int = _READ_TTL_SECONDS,
+ ) -> None:
+ self._prisma_client_source = prisma_client_source
+ self._sink = sink
+ self._read_ttl_seconds = read_ttl_seconds
+ self._readings: LocalTTLCache = InMemoryCache(max_size_in_memory=10_000, default_ttl=read_ttl_seconds)
+
+ async def read(self, spend_key: str, budget_duration: str) -> SharedWindow | None:
+ cached: Final = self._readings.get_cache(key=spend_key)
+ if isinstance(cached, _Reading):
+ return cached.window
+ window: Final = await self._read_row(spend_key=spend_key, budget_duration=budget_duration)
+ self._readings.set_cache(key=spend_key, value=_Reading(window=window), ttl=self._read_ttl_seconds)
+ return window
+
+ async def add(self, spend_key: str, budget_duration: str, started_at: float, spend: float) -> None:
+ if self._prisma_client_source() is None:
+ return
+ try:
+ await self._sink(
+ build_window_spend_transaction(
+ entity_type=MODEL_BUDGET_WINDOW_ENTITY_TYPE,
+ entity_id=spend_key,
+ window_duration=budget_duration,
+ window_start=datetime.fromtimestamp(started_at, tz=timezone.utc),
+ spend=spend,
+ )
+ )
+ except Exception: # noqa: BLE001 # spend tracking must never fail the success callback
+ verbose_proxy_logger.debug("Unable to queue shared model budget spend for %s", spend_key, exc_info=True)
+
+ async def _read_row(self, spend_key: str, budget_duration: str) -> SharedWindow | None:
+ prisma_client: Final = self._prisma_client_source()
+ if prisma_client is None:
+ return None
+ try:
+ row: Final = await BudgetWindowSpendRepository(prisma_client).table.find_unique(
+ where={
+ "entity_type_entity_id_window_duration": {
+ "entity_type": MODEL_BUDGET_WINDOW_ENTITY_TYPE,
+ "entity_id": spend_key,
+ "window_duration": budget_duration,
+ }
+ }
+ )
+ except Exception: # noqa: BLE001 # an unreadable row leaves enforcement on this replica's own counter
+ verbose_proxy_logger.exception("Unable to read shared model budget spend for %s", spend_key)
+ return None
+ if row is None:
+ return None
+ window_start: Final = row.window_start
+ started_at: Final = (
+ window_start if window_start.tzinfo is not None else window_start.replace(tzinfo=timezone.utc)
+ ).timestamp()
+ return SharedWindow(started_at=started_at, spend=float(row.spend or 0.0))
+
+
+PROXY_SHARED_WINDOWS: Final = DatabaseSharedWindows()
diff --git a/litellm/proxy/hooks/model_max_budget_limiter.py b/litellm/proxy/hooks/model_max_budget_limiter.py
index 37cef41034b..d742e8148a7 100644
--- a/litellm/proxy/hooks/model_max_budget_limiter.py
+++ b/litellm/proxy/hooks/model_max_budget_limiter.py
@@ -1,11 +1,13 @@
import asyncio
+import functools
import json
+import re
import time
-from collections.abc import Callable, Iterable, Mapping, Sequence
+from collections.abc import Callable, Collection, Iterable, Iterator, Mapping, Sequence
from dataclasses import dataclass
from datetime import datetime, timezone
from types import MappingProxyType
-from typing import Final, cast # noqa: TID251 # untyped callback payload requires explicit narrowing
+from typing import Final, TypeAlias, cast # noqa: TID251 # untyped callback payload requires explicit narrowing
import litellm
from litellm._internal_context import with_service_target
@@ -16,8 +18,15 @@ from litellm.litellm_core_utils.duration_parser import duration_in_seconds
from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
from litellm.llms.bedrock.common_utils import get_bedrock_base_model
from litellm.proxy._types import Litellm_EntityType, UserAPIKeyAuth
+from litellm.proxy.hooks.model_budget_shared_windows import (
+ PROXY_SHARED_WINDOWS,
+ LocalTTLCache,
+ SharedWindow,
+ SharedWindows,
+)
from litellm.router_strategy.budget_limiter import RouterBudgetLimiting, budget_window_seconds
from litellm.router_utils.batch_utils import batch_charged_once_marker_key, batch_id_to_charge_once
+from litellm.router_utils.pattern_match_deployments import PatternUtils
from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import BudgetConfig, StandardLoggingPayload
@@ -43,6 +52,10 @@ _PROCESS_STARTED_AT: Final = time.monotonic()
_LATEST_RESET_AT: Final = datetime(9999, 12, 31, tzinfo=timezone.utc).timestamp()
+_GLOB_WILDCARD: Final = "*"
+
+ModelAccessGroups: TypeAlias = Callable[[str], Collection[str]]
+
_BUDGET_START_TIME_KEY_PREFIXES: Final = MappingProxyType(
{
Litellm_EntityType.KEY: "virtual_key_budget_start_time",
@@ -131,36 +144,100 @@ def model_budget_start_time_cache_key(
return f"{_BUDGET_START_TIME_KEY_PREFIXES[entity_type]}:{entity_id}:{budget_model}:{budget_duration}"
-def resolve_model_budget(model: str, model_max_budget: Mapping[str, object]) -> ResolvedModelBudget | None:
- """Find the `model_max_budget` entry that governs `model`, or None."""
- for candidate in _budget_model_candidates(model):
- raw_budget_config = model_max_budget.get(candidate)
- if raw_budget_config is None:
- continue
- if (budget_config := _usable_budget_config(raw_budget_config)) is None:
- # An entry that will not validate cannot be keyed, so it cannot be
- # enforced or incremented. Skip to the next candidate rather than
- # raising: raising would abort every other scope's increment and turn
- # a config typo into a 500, and stopping here would let one malformed
- # specific entry disable a perfectly good bare-family budget beside
- # it. The candidate chain already falls through an ABSENT entry, and
- # an unparseable one is indistinguishable from absent to enforcement.
- # `validate_model_max_budget` rejects these on the write path, so
- # reaching here means config.yaml or a direct DB edit.
- verbose_proxy_logger.warning(
- "Ignoring unusable model_max_budget entry for %s; it cannot be enforced or tracked",
- candidate,
- )
- continue
- return ResolvedModelBudget(budget_model=candidate, budget_config=budget_config)
- return None
+def proxy_model_access_groups(model: str) -> frozenset[str]:
+ try:
+ from litellm.proxy.proxy_server import llm_router
+ except ImportError:
+ return frozenset()
+ if llm_router is None:
+ return frozenset()
+ return frozenset(llm_router.get_model_access_groups(model_name=model))
-def team_model_budget_applies(model: str, key_model_max_budget: Mapping[str, object] | None) -> bool:
+def resolve_model_budget(
+ model: str,
+ model_max_budget: Mapping[str, object],
+ model_access_groups: ModelAccessGroups = proxy_model_access_groups,
+) -> ResolvedModelBudget | None:
+ """Find the `model_max_budget` entry that governs `model`, or None.
+
+ An entry named after the model wins, then one named after a model access
+ group serving it, then the most specific glob (`claude-opus*`). Every model
+ an access group or glob entry covers draws from that entry's one counter.
+ """
+ resolved: Final = (
+ _usable_entry(entry=entry, raw_budget_config=model_max_budget[entry])
+ for entry in _governing_entry_names(
+ model=model, model_max_budget=model_max_budget, model_access_groups=model_access_groups
+ )
+ )
+ return next((entry for entry in resolved if entry is not None), None)
+
+
+def _governing_entry_names(
+ model: str,
+ model_max_budget: Mapping[str, object],
+ model_access_groups: ModelAccessGroups,
+) -> Iterator[str]:
+ candidates: Final = _budget_model_candidates(model)
+ yield from (candidate for candidate in candidates if candidate in model_max_budget)
+ group_entries: Final = tuple(
+ sorted(entry for entry in model_max_budget if _GLOB_WILDCARD not in entry and entry not in candidates)
+ )
+ if group_entries:
+ groups: Final = model_access_groups(model)
+ yield from (entry for entry in group_entries if entry in groups)
+ yield from sorted(
+ (entry for entry in model_max_budget if _GLOB_WILDCARD in entry and _glob_matches(entry, candidates)),
+ key=_glob_precedence,
+ reverse=True,
+ )
+
+
+def _glob_matches(pattern: str, names: Iterable[str]) -> bool:
+ regex: Final = _glob_regex(pattern)
+ return any(regex.fullmatch(name) for name in names)
+
+
+@functools.lru_cache(maxsize=1024)
+def _glob_regex(pattern: str) -> re.Pattern[str]:
+ return re.compile(re.escape(pattern).replace(re.escape(_GLOB_WILDCARD), ".*"))
+
+
+def _glob_precedence(pattern: str) -> tuple[tuple[int, int], str]:
+ return PatternUtils.calculate_pattern_specificity(pattern), pattern
+
+
+def _usable_entry(entry: str, raw_budget_config: object) -> ResolvedModelBudget | None:
+ if (budget_config := _usable_budget_config(raw_budget_config)) is None:
+ # An entry that will not validate cannot be keyed, so it cannot be
+ # enforced or incremented. Skip to the next candidate rather than
+ # raising: raising would abort every other scope's increment and turn
+ # a config typo into a 500, and stopping here would let one malformed
+ # specific entry disable a perfectly good bare-family budget beside
+ # it. The candidate chain already falls through an ABSENT entry, and
+ # an unparseable one is indistinguishable from absent to enforcement.
+ # `validate_model_max_budget` rejects these on the write path, so
+ # reaching here means config.yaml or a direct DB edit.
+ verbose_proxy_logger.warning(
+ "Ignoring unusable model_max_budget entry for %s; it cannot be enforced or tracked",
+ entry,
+ )
+ return None
+ return ResolvedModelBudget(budget_model=entry, budget_config=budget_config)
+
+
+def team_model_budget_applies(
+ model: str,
+ key_model_max_budget: Mapping[str, object] | None,
+ model_access_groups: ModelAccessGroups = proxy_model_access_groups,
+) -> bool:
"""A key entry that spend-gates `model` overrides the team cap: it is then gated on and billed to the key alone."""
if not key_model_max_budget:
return True
- resolved: Final = resolve_model_budget(model=model, model_max_budget=key_model_max_budget)
+ resolved: Final = resolve_model_budget(
+ model=model, model_max_budget=key_model_max_budget, model_access_groups=model_access_groups
+ )
return resolved is None or not _spend_gated(resolved.budget_config)
@@ -208,6 +285,7 @@ async def build_model_max_budget_usage(
model_max_budget: Mapping[str, object] | None,
cache: DualCache | None,
now: datetime | None = None,
+ shared_windows: SharedWindows = PROXY_SHARED_WINDOWS,
) -> dict[str, dict[str, object]]:
"""Current-window spend per configured budget model, as `/key/info` and `/user/info` report it.
@@ -231,17 +309,18 @@ async def build_model_max_budget_usage(
)
if not budgets:
return {}
+ spend_keys: Final = tuple(
+ model_budget_spend_cache_key(
+ entity_type=entity_type,
+ entity_id=entity_id,
+ budget_model=budget_model,
+ budget_duration=budget_config.budget_duration,
+ )
+ for budget_model, budget_config in budgets
+ )
windows: Final = await _read_windows(
cache=cache,
- spend_keys=tuple(
- model_budget_spend_cache_key(
- entity_type=entity_type,
- entity_id=entity_id,
- budget_model=budget_model,
- budget_duration=budget_config.budget_duration,
- )
- for budget_model, budget_config in budgets
- ),
+ spend_keys=spend_keys,
start_time_keys=tuple(
model_budget_start_time_cache_key(
entity_type=entity_type,
@@ -253,21 +332,57 @@ async def build_model_max_budget_usage(
),
)
current_time: Final = (now or datetime.now(timezone.utc)).timestamp()
+ shared: Final = tuple(
+ [
+ (
+ await _open_shared_window(
+ shared_windows=shared_windows,
+ spend_key=spend_key,
+ budget_duration=budget_config.budget_duration,
+ now=current_time,
+ )
+ if cache.redis_cache is None
+ else None
+ )
+ for spend_key, (_, budget_config) in zip(spend_keys, budgets, strict=True)
+ ]
+ )
+ readings: Final = tuple(
+ _fuller_reading(local=window, shared=shared_window)
+ for window, shared_window in zip(windows, shared, strict=True)
+ )
return {
budget_model: {
- "current_spend": round(window.current_spend, 4),
+ "current_spend": round(reading.current_spend, 4),
"budget_limit": budget_config.max_budget,
"time_period": budget_config.budget_duration,
"budget_reset_at": _window_reset_at(
- started_at=window.started_at,
+ started_at=reading.started_at,
budget_duration=budget_config.budget_duration,
now=current_time,
),
}
- for (budget_model, budget_config), window in zip(budgets, windows, strict=True)
+ for (budget_model, budget_config), reading in zip(budgets, readings, strict=True)
}
+async def _open_shared_window(
+ shared_windows: SharedWindows, spend_key: str, budget_duration: str | None, now: float
+) -> SharedWindow | None:
+ if budget_duration is None:
+ return None
+ window: Final = await shared_windows.read(spend_key=spend_key, budget_duration=budget_duration)
+ if window is None or not window.is_open(budget_duration=budget_duration, now=now):
+ return None
+ return window
+
+
+def _fuller_reading(local: _WindowReading, shared: SharedWindow | None) -> _WindowReading:
+ if shared is None or shared.spend <= local.current_spend:
+ return local
+ return _WindowReading(current_spend=shared.spend, started_at=shared.started_at)
+
+
@with_service_target("model_budgets")
async def _read_windows(
cache: DualCache, spend_keys: Sequence[str], start_time_keys: Sequence[str]
@@ -337,13 +452,18 @@ def _as_float(value: object) -> float | None:
def _resolve_entity_model_budgets(
model: str,
entity_budgets: Iterable[tuple[Litellm_EntityType, str | None, object]],
+ model_access_groups: ModelAccessGroups,
) -> tuple[tuple[Litellm_EntityType, str, ResolvedModelBudget], ...]:
"""Drop the scopes that do not budget `model`, keeping only what can be incremented."""
return tuple(
(entity_type, entity_id, resolved)
for entity_type, entity_id, model_max_budget in entity_budgets
if entity_id is not None and isinstance(model_max_budget, Mapping) and model_max_budget
- for resolved in (resolve_model_budget(model=model, model_max_budget=model_max_budget),)
+ for resolved in (
+ resolve_model_budget(
+ model=model, model_max_budget=model_max_budget, model_access_groups=model_access_groups
+ ),
+ )
if resolved is not None and resolved.budget_config.budget_duration is not None
)
@@ -367,9 +487,17 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
Example: key=$LITELLM_API_KEY, model=gpt-4o, max_budget=100, time_period=1d
"""
- def __init__(self, dual_cache: DualCache, clock: Callable[[], float] = time.time):
+ def __init__(
+ self,
+ dual_cache: DualCache,
+ clock: Callable[[], float] = time.time,
+ model_access_groups: ModelAccessGroups = proxy_model_access_groups,
+ shared_windows: SharedWindows = PROXY_SHARED_WINDOWS,
+ ):
self.dual_cache = dual_cache
self._clock = clock
+ self._model_access_groups = model_access_groups
+ self._shared_windows = shared_windows
self.redis_increment_operation_queue = []
self._redis_increment_queue_lock = asyncio.Lock()
self._redis_increment_flush_lock = asyncio.Lock()
@@ -471,7 +599,9 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
Raises:
BudgetExceededError: If the team has exceeded the model budget
"""
- if not team_model_budget_applies(model=model, key_model_max_budget=key_model_max_budget):
+ if not team_model_budget_applies(
+ model=model, key_model_max_budget=key_model_max_budget, model_access_groups=self._model_access_groups
+ ):
return True
return await self._is_entity_within_model_budget(
entity_type=Litellm_EntityType.TEAM,
@@ -509,7 +639,9 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
) -> bool:
if not model_max_budget:
return True
- resolved: Final = resolve_model_budget(model=model, model_max_budget=model_max_budget)
+ resolved: Final = resolve_model_budget(
+ model=model, model_max_budget=model_max_budget, model_access_groups=self._model_access_groups
+ )
if resolved is None:
verbose_proxy_logger.debug("Model %s not found in %s model_max_budget", model, entity_type.value)
return True
@@ -523,6 +655,7 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
entity_id=entity_id,
model=model,
resolved=resolved,
+ max_budget=max_budget,
)
if current_spend >= max_budget:
raise litellm.BudgetExceededError(
@@ -540,12 +673,16 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
entity_id: str | None,
model: str,
resolved: ResolvedModelBudget,
+ max_budget: float,
) -> float:
"""Spend charged to this budget in the current window, legacy counter included.
A counter that was never written is zero spend, not unknown spend. The
distinction only shows up at a zero-dollar cap, where skipping the
comparison would let the strictest possible limit admit every request.
+
+ Without Redis each replica's counter holds only its own share, so the
+ window every replica adds to in the database is the floor.
"""
spend_key: Final = model_budget_spend_cache_key(
entity_type=entity_type,
@@ -559,10 +696,21 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
model=model,
resolved=resolved,
)
- current_spend: Final = _as_spend(await self._cached_spend(spend_key))
- if legacy_spend_key is None or legacy_spend_key == spend_key:
- return current_spend
- return current_spend + _as_spend(await self._cached_spend(legacy_spend_key))
+ counted: Final = _as_spend(await self._cached_spend(spend_key))
+ local_spend: Final = (
+ counted
+ if legacy_spend_key is None or legacy_spend_key == spend_key
+ else counted + _as_spend(await self._cached_spend(legacy_spend_key))
+ )
+ if local_spend >= max_budget or self.dual_cache.redis_cache is not None:
+ return local_spend
+ shared: Final = await _open_shared_window(
+ shared_windows=self._shared_windows,
+ spend_key=spend_key,
+ budget_duration=resolved.budget_config.budget_duration,
+ now=self._clock(),
+ )
+ return local_spend if shared is None else max(local_spend, shared.spend)
async def _cached_spend(self, spend_key: str) -> float | None:
redis_cache: Final = self.dual_cache.redis_cache
@@ -643,6 +791,7 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
key_model_max_budget=(
key_model_max_budget if isinstance(key_model_max_budget, Mapping) else None
),
+ model_access_groups=self._model_access_groups,
)
else None
),
@@ -667,7 +816,9 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
),
)
- resolved_budgets: Final = _resolve_entity_model_budgets(model=model, entity_budgets=entity_budgets)
+ resolved_budgets: Final = _resolve_entity_model_budgets(
+ model=model, entity_budgets=entity_budgets, model_access_groups=self._model_access_groups
+ )
if not resolved_budgets:
verbose_proxy_logger.debug(
"Not running _PROXY_VirtualKeyModelMaxBudgetLimiter.async_log_success_event: "
@@ -721,17 +872,61 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
ttl_seconds=duration_in_seconds(budget_duration),
):
return
+ start_time_key: Final = model_budget_start_time_cache_key(
+ entity_type=entity_type,
+ entity_id=entity_id,
+ budget_model=resolved.budget_model,
+ budget_duration=budget_duration,
+ )
await self._increment_spend_for_key(
budget_config=resolved.budget_config,
spend_key=spend_key,
- start_time_key=model_budget_start_time_cache_key(
- entity_type=entity_type,
- entity_id=entity_id,
- budget_model=resolved.budget_model,
- budget_duration=budget_duration,
- ),
+ start_time_key=start_time_key,
response_cost=response_cost,
)
+ await self._share_spend(
+ spend_key=spend_key,
+ start_time_key=start_time_key,
+ budget_duration=budget_duration,
+ response_cost=response_cost,
+ )
+
+ async def _share_spend(
+ self, spend_key: str, start_time_key: str, budget_duration: str, response_cost: float
+ ) -> None:
+ """Without Redis, add this charge to the window every replica reads.
+
+ A replica joins the window that is already open rather than its own, so
+ all of them roll over together and none resets the others' spend.
+ """
+ if self.dual_cache.redis_cache is not None:
+ return
+ now: Final = self._clock()
+ shared: Final = await _open_shared_window(
+ shared_windows=self._shared_windows,
+ spend_key=spend_key,
+ budget_duration=budget_duration,
+ now=now,
+ )
+ if shared is not None:
+ _join_window(
+ local=self.dual_cache.in_memory_cache,
+ spend_key=spend_key,
+ start_time_key=start_time_key,
+ window=shared,
+ budget_duration=budget_duration,
+ now=now,
+ )
+ await self._shared_windows.add(
+ spend_key=spend_key,
+ budget_duration=budget_duration,
+ started_at=await self._get_or_set_budget_start_time(
+ start_time_key=start_time_key,
+ current_time=now,
+ ttl_seconds=budget_window_seconds(now, budget_duration),
+ ),
+ spend=response_cost,
+ )
async def _claim_batch_charge(self, spend_key: str, batch_id: str, ttl_seconds: int) -> bool:
marker_key: Final = batch_charged_once_marker_key(spend_key=spend_key, batch_id=batch_id)
@@ -741,4 +936,14 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
return polls == 1
+def _join_window(
+ local: LocalTTLCache, spend_key: str, start_time_key: str, window: SharedWindow, budget_duration: str, now: float
+) -> None:
+ remaining: Final = window.started_at + budget_window_seconds(window.started_at, budget_duration) - now
+ local.set_cache(key=start_time_key, value=window.started_at, ttl=remaining)
+ counted: Final = local.get_cache(key=spend_key)
+ if counted is not None:
+ local.set_cache(key=spend_key, value=counted, ttl=remaining)
+
+
PROXY_VirtualKeyModelMaxBudgetLimiter = _PROXY_VirtualKeyModelMaxBudgetLimiter
diff --git a/tests/unit/proxy/hooks/test_model_budget_shared_windows.py b/tests/unit/proxy/hooks/test_model_budget_shared_windows.py
new file mode 100644
index 00000000000..70957b1f74d
--- /dev/null
+++ b/tests/unit/proxy/hooks/test_model_budget_shared_windows.py
@@ -0,0 +1,93 @@
+from datetime import datetime, timezone
+from types import SimpleNamespace
+from typing import Final
+
+import pytest
+
+from litellm.proxy.db.db_transaction_queue.window_spend_update_queue import WindowSpendTransaction
+from litellm.proxy.hooks.model_budget_shared_windows import DatabaseSharedWindows, SharedWindow
+
+SPEND_KEY: Final = "virtual_key_spend:key-hash:claude-opus*:1d"
+WINDOW_START: Final = datetime(2026, 10, 20, 9, 30, tzinfo=timezone.utc)
+
+
+class _Table:
+ def __init__(self, row: object | None = None, error: Exception | None = None) -> None:
+ self.row = row
+ self.error = error
+ self.reads = 0
+
+ async def find_unique(self, where: object) -> object | None:
+ self.reads = self.reads + 1
+ if self.error is not None:
+ raise self.error
+ return self.row
+
+
+class _Sink:
+ def __init__(self) -> None:
+ self.transactions: tuple[WindowSpendTransaction, ...] = ()
+
+ async def __call__(self, transaction: WindowSpendTransaction) -> None:
+ self.transactions = (*self.transactions, transaction)
+
+
+def _database_windows(table: _Table | None, sink: _Sink) -> DatabaseSharedWindows:
+ prisma: Final = None if table is None else SimpleNamespace(db=SimpleNamespace(litellm_budgetwindowspend=table))
+ return DatabaseSharedWindows(
+ prisma_client_source=lambda: prisma, # pyright: ignore[reportArgumentType] # duck-typed prisma double
+ sink=sink,
+ )
+
+
+def test_a_window_is_open_until_its_period_has_passed():
+ window: Final = SharedWindow(started_at=WINDOW_START.timestamp(), spend=1.0)
+ day: Final = 24 * 60 * 60
+
+ assert window.is_open(budget_duration="1d", now=WINDOW_START.timestamp() + day - 1)
+ assert not window.is_open(budget_duration="1d", now=WINDOW_START.timestamp() + day)
+
+
+@pytest.mark.asyncio
+async def test_read_returns_the_stored_window_and_rereads_only_after_the_cache_expires():
+ table: Final = _Table(row=SimpleNamespace(window_start=WINDOW_START.replace(tzinfo=None), spend=2.5))
+ windows: Final = _database_windows(table, _Sink())
+
+ first: Final = await windows.read(spend_key=SPEND_KEY, budget_duration="1d")
+ second: Final = await windows.read(spend_key=SPEND_KEY, budget_duration="1d")
+
+ assert first == SharedWindow(started_at=WINDOW_START.timestamp(), spend=2.5)
+ assert second == first
+ assert table.reads == 1
+
+
+@pytest.mark.asyncio
+async def test_an_unreadable_table_reads_as_no_shared_window():
+ table: Final = _Table(error=RuntimeError("database is down"))
+ windows: Final = _database_windows(table, _Sink())
+
+ assert await windows.read(spend_key=SPEND_KEY, budget_duration="1d") is None
+
+
+@pytest.mark.asyncio
+async def test_add_queues_the_charge_for_the_window_it_belongs_to():
+ sink: Final = _Sink()
+ windows: Final = _database_windows(_Table(), sink)
+
+ await windows.add(spend_key=SPEND_KEY, budget_duration="1d", started_at=WINDOW_START.timestamp(), spend=0.25)
+
+ assert len(sink.transactions) == 1
+ queued: Final = sink.transactions[0]
+ assert (queued["entity_id"], queued["window_duration"], queued["spend"]) == (SPEND_KEY, "1d", 0.25)
+ assert datetime.fromisoformat(queued["window_start"]).replace(tzinfo=timezone.utc) == WINDOW_START
+
+
+@pytest.mark.asyncio
+async def test_nothing_is_queued_without_a_database():
+ sink: Final = _Sink()
+ windows: Final = _database_windows(None, sink)
+
+ await windows.add(spend_key=SPEND_KEY, budget_duration="1d", started_at=WINDOW_START.timestamp(), spend=0.25)
+
+ assert sink.transactions == ()
+ assert await windows.read(spend_key=SPEND_KEY, budget_duration="1d") is None
diff --git a/tests/unit/proxy/hooks/test_model_max_budget_limiter.py b/tests/unit/proxy/hooks/test_model_max_budget_limiter.py
index 2678f4205d5..33fa7bc3daf 100644
--- a/tests/unit/proxy/hooks/test_model_max_budget_limiter.py
+++ b/tests/unit/proxy/hooks/test_model_max_budget_limiter.py
@@ -8,10 +8,13 @@ import pytest
import litellm
from litellm.caching.caching import DualCache
-from litellm.proxy._types import Litellm_EntityType
+from litellm.proxy._types import Litellm_EntityType, UserAPIKeyAuth
+from litellm.proxy.hooks.model_budget_shared_windows import SharedWindow, SharedWindows
from litellm.proxy.hooks.model_max_budget_limiter import (
PROXY_VirtualKeyModelMaxBudgetLimiter,
+ build_model_max_budget_usage,
model_budget_spend_cache_key,
+ resolve_model_budget,
team_member_budget_entity_id,
)
from litellm.types.caching import RedisPipelineIncrementOperation
@@ -74,10 +77,14 @@ def _event(
}
+async def _log_success(
+ limiter: PROXY_VirtualKeyModelMaxBudgetLimiter, event: dict[str, object], response_obj: object = None
+) -> None:
+ await limiter.async_log_success_event(event, response_obj=response_obj, start_time=None, end_time=None)
+
+
async def _poll(limiter: PROXY_VirtualKeyModelMaxBudgetLimiter, batch: LiteLLMBatch, response_cost: float) -> None:
- await limiter.async_log_success_event(
- _event("aretrieve_batch", response_cost), response_obj=batch, start_time=None, end_time=None
- )
+ await _log_success(limiter, _event("aretrieve_batch", response_cost), response_obj=batch)
async def _chat(
@@ -86,7 +93,8 @@ async def _chat(
user_id: str = USER_ID,
team_id: str = TEAM_ID,
) -> None:
- await limiter.async_log_success_event(
+ await _log_success(
+ limiter,
_event(
"acompletion",
CHAT_COST,
@@ -94,9 +102,6 @@ async def _chat(
user_id=user_id,
team_id=team_id,
),
- response_obj=None,
- start_time=None,
- end_time=None,
)
@@ -311,3 +316,239 @@ async def test_a_batch_polled_within_every_budget_window_is_never_charged_again(
await _poll(limiter, finished, BATCH_COST)
assert _local_spend(limiter, KEY_SPEND_KEY) == pytest.approx(BATCH_COST)
+
+
+OPUS_GROUP: Final = "claude-opus"
+OPUS_GROUP_MEMBERS: Final = MappingProxyType(
+ {"claude-opus-4-8": frozenset({OPUS_GROUP}), "claude-opus-5-5": frozenset({OPUS_GROUP})}
+)
+FAMILY_CAP: Final = {"budget_limit": CHAT_COST, "time_period": "1d"} # mutable-ok: BudgetConfig validates dicts only
+ROOMY_CAP: Final = {"budget_limit": CHAT_COST * 100, "time_period": "1d"} # mutable-ok: BudgetConfig validates dicts only
+
+
+def _model_access_groups(groups_by_model: Mapping[str, frozenset[str]]) -> Callable[[str], frozenset[str]]:
+ return lambda model: groups_by_model.get(model, frozenset())
+
+
+class _SharedWindowsDouble:
+ def __init__(self) -> None:
+ self.rows: Mapping[str, SharedWindow] = MappingProxyType({})
+
+ async def read(self, spend_key: str, budget_duration: str) -> SharedWindow | None:
+ return self.rows.get(spend_key)
+
+ async def add(self, spend_key: str, budget_duration: str, started_at: float, spend: float) -> None:
+ stored: Final = self.rows.get(spend_key)
+ row: Final = (
+ SharedWindow(started_at=stored.started_at, spend=stored.spend + spend)
+ if stored is not None and stored.started_at >= started_at
+ else SharedWindow(started_at=started_at, spend=spend)
+ )
+ self.rows = MappingProxyType({**self.rows, spend_key: row})
+
+
+def _family_limiter(
+ groups_by_model: Mapping[str, frozenset[str]] = OPUS_GROUP_MEMBERS,
+ dual_cache: DualCache | None = None,
+ shared_windows: SharedWindows | None = None,
+ clock: Callable[[], float] = time.time,
+) -> PROXY_VirtualKeyModelMaxBudgetLimiter:
+ return PROXY_VirtualKeyModelMaxBudgetLimiter(
+ dual_cache=dual_cache or DualCache(),
+ clock=clock,
+ model_access_groups=_model_access_groups(groups_by_model),
+ shared_windows=shared_windows or _SharedWindowsDouble(),
+ )
+
+
+def _key(model_max_budget: Mapping[str, object]) -> UserAPIKeyAuth:
+ return UserAPIKeyAuth(token=KEY_HASH, key_alias="opus-family", model_max_budget=dict(model_max_budget))
+
+
+async def _spend_on(
+ limiter: PROXY_VirtualKeyModelMaxBudgetLimiter, model: str, model_max_budget: Mapping[str, object]
+) -> None:
+ await _log_success(
+ limiter,
+ {
+ "call_type": "acompletion",
+ "standard_logging_object": {
+ "call_type": "acompletion",
+ "response_cost": CHAT_COST,
+ "model": f"anthropic/{model}",
+ "model_group": model,
+ "metadata": {"user_api_key_hash": KEY_HASH},
+ },
+ "litellm_params": {"metadata": {"user_api_key_model_max_budget": dict(model_max_budget)}},
+ },
+ )
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("family_entry", ["claude-opus*", OPUS_GROUP])
+async def test_every_opus_version_draws_from_one_family_entry(family_entry: str):
+ budget: Final = {family_entry: FAMILY_CAP}
+ limiter: Final = _family_limiter()
+ assert await limiter.is_key_within_model_budget(user_api_key_dict=_key(budget), model="claude-opus-5-5")
+
+ await _spend_on(limiter, model="claude-opus-4-8", model_max_budget=budget)
+
+ with pytest.raises(litellm.BudgetExceededError, match="model=claude-opus-5-5"):
+ await limiter.is_key_within_model_budget(user_api_key_dict=_key(budget), model="claude-opus-5-5")
+ assert await limiter.is_key_within_model_budget(user_api_key_dict=_key(budget), model="claude-sonnet-5")
+
+
+@pytest.mark.asyncio
+async def test_a_model_added_to_the_access_group_later_is_covered_with_no_key_change():
+ budget: Final = {OPUS_GROUP: FAMILY_CAP}
+ shared_cache: Final = DualCache()
+ before_group_change: Final = _family_limiter(dual_cache=shared_cache)
+ after_group_change: Final = _family_limiter(
+ groups_by_model={**OPUS_GROUP_MEMBERS, "claude-opus-6": frozenset({OPUS_GROUP})}, dual_cache=shared_cache
+ )
+
+ await _spend_on(before_group_change, model="claude-opus-4-8", model_max_budget=budget)
+
+ assert await before_group_change.is_key_within_model_budget(user_api_key_dict=_key(budget), model="claude-opus-6")
+ with pytest.raises(litellm.BudgetExceededError):
+ await after_group_change.is_key_within_model_budget(user_api_key_dict=_key(budget), model="claude-opus-6")
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+ ("budget", "checked_model", "blocked"),
+ [
+ ({"claude-opus-4-8": ROOMY_CAP, "claude-opus*": FAMILY_CAP}, "claude-opus-4-8", False),
+ ({"claude-opus-4-8": ROOMY_CAP, "claude-opus*": FAMILY_CAP}, "claude-opus-5-5", False),
+ ({OPUS_GROUP: FAMILY_CAP, "claude-opus*": ROOMY_CAP}, "claude-opus-5-5", True),
+ ({"claude-opus-4*": FAMILY_CAP, "claude-opus*": ROOMY_CAP}, "claude-opus-4-7", True),
+ ({"claude-opus-4*": FAMILY_CAP, "claude-opus*": ROOMY_CAP}, "claude-opus-5-5", False),
+ ({"claude-opus-4*": {"budget_limit": CHAT_COST}, "claude-opus*": FAMILY_CAP}, "claude-opus-5-5", True),
+ ],
+)
+async def test_the_most_specific_entry_takes_the_spend_and_gates_the_request(
+ budget: Mapping[str, object], checked_model: str, blocked: bool
+):
+ limiter: Final = _family_limiter()
+
+ await _spend_on(limiter, model="claude-opus-4-8", model_max_budget=budget)
+
+ if blocked:
+ with pytest.raises(litellm.BudgetExceededError):
+ await limiter.is_key_within_model_budget(user_api_key_dict=_key(budget), model=checked_model)
+ else:
+ assert await limiter.is_key_within_model_budget(user_api_key_dict=_key(budget), model=checked_model)
+
+
+@pytest.mark.asyncio
+async def test_key_info_usage_reports_the_family_total():
+ budget: Final = {"claude-opus*": ROOMY_CAP}
+ limiter: Final = _family_limiter()
+
+ await _spend_on(limiter, model="claude-opus-4-8", model_max_budget=budget)
+ await _spend_on(limiter, model="claude-opus-5-5", model_max_budget=budget)
+
+ usage: Final = await build_model_max_budget_usage(
+ entity_type=Litellm_EntityType.KEY,
+ entity_id=KEY_HASH,
+ model_max_budget=budget,
+ cache=limiter.dual_cache,
+ )
+ assert usage["claude-opus*"]["current_spend"] == round(CHAT_COST * 2, 4)
+
+
+@pytest.mark.parametrize(
+ ("pattern", "model", "governs"),
+ [
+ ("claude-opus*", "claude-opus-4-8", True),
+ ("claude-opus*", "anthropic/claude-opus-4-8", True),
+ ("anthropic/claude-opus*", "anthropic/claude-opus-4-8", True),
+ ("anthropic/claude-opus*", "claude-opus-4-8", False),
+ ("claude-opus*", "my-claude-opus-4-8", False),
+ ("claude-opus*", "claude-sonnet-5", False),
+ ("gpt-4.1*", "gpt-401-mini", False),
+ ("*", "claude-sonnet-5", True),
+ ],
+)
+def test_glob_entries_match_the_whole_model_name_with_only_star_as_a_wildcard(
+ pattern: str, model: str, governs: bool
+):
+ resolved: Final = resolve_model_budget(
+ model=model, model_max_budget={pattern: FAMILY_CAP}, model_access_groups=_model_access_groups({})
+ )
+
+ assert (resolved is not None and resolved.budget_model == pattern) is governs
+
+
+@pytest.mark.asyncio
+async def test_replicas_without_redis_enforce_one_family_cap_together():
+ budget: Final = {"claude-opus*": FAMILY_CAP}
+ table: Final = _SharedWindowsDouble()
+ replica_a: Final = _family_limiter(shared_windows=table)
+ replica_b: Final = _family_limiter(shared_windows=table)
+
+ await _spend_on(replica_a, model="claude-opus-4-8", model_max_budget=budget)
+
+ with pytest.raises(litellm.BudgetExceededError, match="model=claude-opus-5-5"):
+ await replica_b.is_key_within_model_budget(user_api_key_dict=_key(budget), model="claude-opus-5-5")
+
+
+@pytest.mark.asyncio
+async def test_replicas_sharing_redis_enforce_the_family_cap_without_the_database():
+ budget: Final = {"claude-opus*": FAMILY_CAP}
+ redis: Final = _SharedRedisDouble()
+ table: Final = _SharedWindowsDouble()
+ replica_a: Final = _family_limiter(
+ dual_cache=DualCache(redis_cache=redis), # pyright: ignore[reportArgumentType] # duck-typed Redis double
+ shared_windows=table,
+ )
+ replica_b: Final = _family_limiter(
+ dual_cache=DualCache(redis_cache=redis), # pyright: ignore[reportArgumentType] # duck-typed Redis double
+ shared_windows=table,
+ )
+
+ await _spend_on(replica_a, model="claude-opus-4-8", model_max_budget=budget)
+ await _drain_redis_pushes()
+
+ with pytest.raises(litellm.BudgetExceededError, match="model=claude-opus-5-5"):
+ await replica_b.is_key_within_model_budget(user_api_key_dict=_key(budget), model="claude-opus-5-5")
+ assert dict(table.rows) == {}
+
+
+@pytest.mark.asyncio
+async def test_a_replica_charges_the_open_shared_window_instead_of_restarting_it():
+ budget: Final = {"claude-opus*": {"budget_limit": CHAT_COST * 1.5, "time_period": "1d"}}
+ table: Final = _SharedWindowsDouble()
+ clock: Final = _Clock()
+ clock.advance(time.time())
+ replica_a: Final = _family_limiter(shared_windows=table, clock=clock.now)
+ replica_b: Final = _family_limiter(shared_windows=table, clock=clock.now)
+
+ await _spend_on(replica_a, model="claude-opus-4-8", model_max_budget=budget)
+ clock.advance(3600)
+ await _spend_on(replica_b, model="claude-opus-5-5", model_max_budget=budget)
+
+ for replica in (replica_a, replica_b):
+ with pytest.raises(litellm.BudgetExceededError):
+ await replica.is_key_within_model_budget(user_api_key_dict=_key(budget), model="claude-opus-5-5")
+
+
+@pytest.mark.asyncio
+async def test_key_info_on_one_replica_reports_spend_from_every_replica():
+ budget: Final = {"claude-opus*": ROOMY_CAP}
+ table: Final = _SharedWindowsDouble()
+ replica_a: Final = _family_limiter(shared_windows=table)
+ replica_b: Final = _family_limiter(shared_windows=table)
+
+ await _spend_on(replica_a, model="claude-opus-4-8", model_max_budget=budget)
+ await _spend_on(replica_b, model="claude-opus-5-5", model_max_budget=budget)
+
+ usage: Final = await build_model_max_budget_usage(
+ entity_type=Litellm_EntityType.KEY,
+ entity_id=KEY_HASH,
+ model_max_budget=budget,
+ cache=replica_a.dual_cache,
+ shared_windows=table,
+ )
+ assert usage["claude-opus*"]["current_spend"] == round(CHAT_COST * 2, 4)
+ assert usage["claude-opus*"]["budget_reset_at"] is not None
From 65c33671833cc944147a2b59c1f7e42263342c17 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 20 Oct 2026 20:34:13 -0700
Subject: [PATCH 3/6] fix(proxy): open each shared model budget window once for
every replica, and match patterns in linear time
Without Redis, the first replica to charge an entry in a new window now opens it in the window spend table and
the others join it, so no flush replaces spend another replica already added. Budget patterns match by splitting
on `*` instead of a regular expression, a refused request names the entry that refused it, and the key form no
longer offers a typed pattern another row already budgets.
---
litellm/integrations/otel/model/spans.py | 1 +
.../proxy/db/budget_window_spend_writer.py | 36 +++++
.../hooks/model_budget_shared_windows.py | 77 +++++++---
.../proxy/hooks/model_max_budget_limiter.py | 133 ++++++++++++------
.../db/test_budget_window_spend_writer.py | 24 ++++
.../hooks/test_model_budget_shared_windows.py | 54 ++++++-
.../hooks/test_model_max_budget_limiter.py | 69 +++++++--
.../test_key_management_endpoints.py | 5 +
tests/unit/test_internal_context.py | 1 +
.../ModelMaxBudgetEditor.integration.test.tsx | 23 ++-
.../key_team_helpers/ModelMaxBudgetEditor.tsx | 3 +-
.../src/components/shared/SearchSelect.tsx | 5 +-
12 files changed, 355 insertions(+), 76 deletions(-)
diff --git a/litellm/integrations/otel/model/spans.py b/litellm/integrations/otel/model/spans.py
index d8be607150c..59519bf3428 100644
--- a/litellm/integrations/otel/model/spans.py
+++ b/litellm/integrations/otel/model/spans.py
@@ -434,6 +434,7 @@ _POSTGRES_OPERATION_BY_CALL_TYPE: Final[Mapping[str, PostgresOperation]] = Mappi
"reset_budget_windows": PostgresOperation("select", None),
"write_budget_windows": PostgresOperation("update", None),
"roll_window_spend_row": PostgresOperation("update", None),
+ "open_window_spend_row": PostgresOperation("upsert", None),
"seed_window_spend": PostgresOperation("select", None),
"select_window_spend_rows": PostgresOperation("select", None),
"commit_window_spend_updates": PostgresOperation("upsert", None),
diff --git a/litellm/proxy/db/budget_window_spend_writer.py b/litellm/proxy/db/budget_window_spend_writer.py
index 82c95cfe998..99b49331932 100644
--- a/litellm/proxy/db/budget_window_spend_writer.py
+++ b/litellm/proxy/db/budget_window_spend_writer.py
@@ -64,6 +64,16 @@ _ROLL_WINDOW_SPEND_SQL: Final = (
"AND window_start < ($4::timestamptz AT TIME ZONE 'UTC')"
)
+_OPEN_WINDOW_SPEND_SQL: Final = (
+ 'INSERT INTO "LiteLLM_BudgetWindowSpend" '
+ "(entity_type, entity_id, window_duration, window_start, spend, created_at, updated_at) "
+ "VALUES ($1, $2, $3, ($4::timestamptz AT TIME ZONE 'UTC'), 0, "
+ "($4::timestamptz AT TIME ZONE 'UTC'), ($4::timestamptz AT TIME ZONE 'UTC')) "
+ "ON CONFLICT (entity_type, entity_id, window_duration) DO UPDATE SET "
+ "window_start = EXCLUDED.window_start, spend = 0, updated_at = EXCLUDED.updated_at "
+ "WHERE \"LiteLLM_BudgetWindowSpend\".window_start <= ($5::timestamptz AT TIME ZONE 'UTC')"
+)
+
_SEED_FROM_SPEND_LOGS_KEY_SQL: Final = (
"SELECT COALESCE(SUM(spend), 0.0) AS total, "
"COALESCE(SUM(spend) FILTER (WHERE \"startTime\" < ($3::timestamptz AT TIME ZONE 'UTC')), 0.0) AS before_batch "
@@ -389,3 +399,29 @@ async def roll_window_spend_row(
to_naive_utc(new_window_start),
to_naive_utc(datetime.now(timezone.utc)),
)
+
+
+async def open_window_spend_row(
+ prisma_client: "PrismaClient",
+ entity_type: str,
+ entity_id: str,
+ window_duration: str,
+ window_start: datetime,
+ replaces_window_start: datetime | None,
+) -> None:
+ """Start a window at `window_start` unless another writer already started one.
+
+ The row is created when it does not exist, and moved onto the new window only
+ while it still holds `replaces_window_start`, the ended window the caller saw.
+ A writer that lost the race leaves the winner's window in place, so reading
+ the row back gives every writer the same window start.
+ """
+ async with db_span("open_window_spend_row", "LiteLLM_BudgetWindowSpend"):
+ await prisma_client.db.execute_raw(
+ _OPEN_WINDOW_SPEND_SQL,
+ entity_type,
+ entity_id,
+ window_duration,
+ to_naive_utc(window_start),
+ None if replaces_window_start is None else to_naive_utc(replaces_window_start),
+ )
diff --git a/litellm/proxy/hooks/model_budget_shared_windows.py b/litellm/proxy/hooks/model_budget_shared_windows.py
index 344114edaa4..56729a33034 100644
--- a/litellm/proxy/hooks/model_budget_shared_windows.py
+++ b/litellm/proxy/hooks/model_budget_shared_windows.py
@@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Final, Protocol, TypeAlias
from litellm._logging import verbose_proxy_logger
from litellm.caching.in_memory_cache import InMemoryCache
+from litellm.proxy.db.budget_window_spend_writer import open_window_spend_row
from litellm.proxy.db.db_transaction_queue.window_spend_update_queue import (
WindowSpendTransaction,
build_window_spend_transaction,
@@ -37,6 +38,8 @@ class LocalTTLCache(Protocol):
class SharedWindows(Protocol):
async def read(self, spend_key: str, budget_duration: str) -> SharedWindow | None: ...
+ async def open(self, spend_key: str, budget_duration: str, now: float) -> float: ...
+
async def add(self, spend_key: str, budget_duration: str, started_at: float, spend: float) -> None: ...
@@ -83,6 +86,40 @@ class DatabaseSharedWindows:
self._readings.set_cache(key=spend_key, value=_Reading(window=window), ttl=self._read_ttl_seconds)
return window
+ async def open(self, spend_key: str, budget_duration: str, now: float) -> float:
+ """When the window every replica shares started, opening one at `now` if none is open.
+
+ Falls back to `now`, this replica's own window, when the database cannot answer.
+ """
+ prisma_client: Final = self._prisma_client_source()
+ if prisma_client is None:
+ return now
+ try:
+ seen: Final = await _fetch_row(
+ prisma_client=prisma_client, spend_key=spend_key, budget_duration=budget_duration
+ )
+ if seen is not None and seen.is_open(budget_duration=budget_duration, now=now):
+ return seen.started_at
+ await open_window_spend_row(
+ prisma_client=prisma_client,
+ entity_type=MODEL_BUDGET_WINDOW_ENTITY_TYPE,
+ entity_id=spend_key,
+ window_duration=budget_duration,
+ window_start=datetime.fromtimestamp(now, tz=timezone.utc),
+ replaces_window_start=None
+ if seen is None
+ else datetime.fromtimestamp(seen.started_at, tz=timezone.utc),
+ )
+ opened: Final = await _fetch_row(
+ prisma_client=prisma_client, spend_key=spend_key, budget_duration=budget_duration
+ )
+ except Exception: # noqa: BLE001 # an unreachable database leaves this replica on its own window
+ verbose_proxy_logger.exception("Unable to open the shared model budget window")
+ return now
+ if opened is None or not opened.is_open(budget_duration=budget_duration, now=now):
+ return now
+ return opened.started_at
+
async def add(self, spend_key: str, budget_duration: str, started_at: float, spend: float) -> None:
if self._prisma_client_source() is None:
return
@@ -97,32 +134,36 @@ class DatabaseSharedWindows:
)
)
except Exception: # noqa: BLE001 # spend tracking must never fail the success callback
- verbose_proxy_logger.debug("Unable to queue shared model budget spend for %s", spend_key, exc_info=True)
+ verbose_proxy_logger.debug("Unable to queue shared model budget spend", exc_info=True)
async def _read_row(self, spend_key: str, budget_duration: str) -> SharedWindow | None:
prisma_client: Final = self._prisma_client_source()
if prisma_client is None:
return None
try:
- row: Final = await BudgetWindowSpendRepository(prisma_client).table.find_unique(
- where={
- "entity_type_entity_id_window_duration": {
- "entity_type": MODEL_BUDGET_WINDOW_ENTITY_TYPE,
- "entity_id": spend_key,
- "window_duration": budget_duration,
- }
- }
- )
+ return await _fetch_row(prisma_client=prisma_client, spend_key=spend_key, budget_duration=budget_duration)
except Exception: # noqa: BLE001 # an unreadable row leaves enforcement on this replica's own counter
- verbose_proxy_logger.exception("Unable to read shared model budget spend for %s", spend_key)
+ verbose_proxy_logger.exception("Unable to read shared model budget spend")
return None
- if row is None:
- return None
- window_start: Final = row.window_start
- started_at: Final = (
- window_start if window_start.tzinfo is not None else window_start.replace(tzinfo=timezone.utc)
- ).timestamp()
- return SharedWindow(started_at=started_at, spend=float(row.spend or 0.0))
+
+
+async def _fetch_row(prisma_client: "PrismaClient", spend_key: str, budget_duration: str) -> SharedWindow | None:
+ row: Final = await BudgetWindowSpendRepository(prisma_client).table.find_unique(
+ where={
+ "entity_type_entity_id_window_duration": {
+ "entity_type": MODEL_BUDGET_WINDOW_ENTITY_TYPE,
+ "entity_id": spend_key,
+ "window_duration": budget_duration,
+ }
+ }
+ )
+ if row is None:
+ return None
+ window_start: Final = row.window_start
+ started_at: Final = (
+ window_start if window_start.tzinfo is not None else window_start.replace(tzinfo=timezone.utc)
+ ).timestamp()
+ return SharedWindow(started_at=started_at, spend=float(row.spend or 0.0))
PROXY_SHARED_WINDOWS: Final = DatabaseSharedWindows()
diff --git a/litellm/proxy/hooks/model_max_budget_limiter.py b/litellm/proxy/hooks/model_max_budget_limiter.py
index d742e8148a7..1d7b7245d8a 100644
--- a/litellm/proxy/hooks/model_max_budget_limiter.py
+++ b/litellm/proxy/hooks/model_max_budget_limiter.py
@@ -1,7 +1,6 @@
import asyncio
import functools
import json
-import re
import time
from collections.abc import Callable, Collection, Iterable, Iterator, Mapping, Sequence
from dataclasses import dataclass
@@ -195,13 +194,29 @@ def _governing_entry_names(
def _glob_matches(pattern: str, names: Iterable[str]) -> bool:
- regex: Final = _glob_regex(pattern)
- return any(regex.fullmatch(name) for name in names)
+ return any(_glob_matches_name(pattern=pattern, name=name) for name in names)
-@functools.lru_cache(maxsize=1024)
-def _glob_regex(pattern: str) -> re.Pattern[str]:
- return re.compile(re.escape(pattern).replace(re.escape(_GLOB_WILDCARD), ".*"))
+def _glob_matches_name(pattern: str, name: str) -> bool:
+ """Whole-name match where `*` is the only wildcard, in linear time whatever the pattern holds.
+
+ Taking each literal piece between two wildcards at its first occurrence is
+ always safe, since a later occurrence only leaves less room for the rest, so
+ the match never needs to backtrack the way a `.*` regex does.
+ """
+ head, *middle, tail = pattern.split(_GLOB_WILDCARD)
+ middle_end: Final = len(name) - len(tail)
+ if middle_end < len(head) or not name.startswith(head) or not name.endswith(tail):
+ return False
+ return functools.reduce(_after_piece(name=name, end=middle_end), middle, len(head)) >= 0
+
+
+def _after_piece(name: str, end: int) -> Callable[[int, str], int]:
+ def step(position: int, piece: str) -> int:
+ found: Final = name.find(piece, position, end) if position >= 0 else -1
+ return found + len(piece) if found >= 0 else -1
+
+ return step
def _glob_precedence(pattern: str) -> tuple[tuple[int, int], str]:
@@ -650,16 +665,26 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
if max_budget is None or max_budget < 0:
return True
- current_spend: Final = await self._get_spend_for_model_budget(
+ local_spend: Final = await self._get_spend_for_model_budget(
entity_type=entity_type,
entity_id=entity_id,
model=model,
resolved=resolved,
- max_budget=max_budget,
+ )
+ current_spend: Final = (
+ local_spend
+ if local_spend >= max_budget
+ else await self._with_shared_spend(
+ entity_type=entity_type, entity_id=entity_id, resolved=resolved, local_spend=local_spend
+ )
)
if current_spend >= max_budget:
raise litellm.BudgetExceededError(
- message=exceeded_message,
+ message=(
+ exceeded_message
+ if resolved.budget_model == model
+ else f"{exceeded_message}, budget_model={resolved.budget_model}"
+ ),
current_cost=current_spend,
max_budget=max_budget,
entity_type=entity_type.value,
@@ -673,16 +698,12 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
entity_id: str | None,
model: str,
resolved: ResolvedModelBudget,
- max_budget: float,
) -> float:
"""Spend charged to this budget in the current window, legacy counter included.
A counter that was never written is zero spend, not unknown spend. The
distinction only shows up at a zero-dollar cap, where skipping the
comparison would let the strictest possible limit admit every request.
-
- Without Redis each replica's counter holds only its own share, so the
- window every replica adds to in the database is the floor.
"""
spend_key: Final = model_budget_spend_cache_key(
entity_type=entity_type,
@@ -696,17 +717,30 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
model=model,
resolved=resolved,
)
- counted: Final = _as_spend(await self._cached_spend(spend_key))
- local_spend: Final = (
- counted
- if legacy_spend_key is None or legacy_spend_key == spend_key
- else counted + _as_spend(await self._cached_spend(legacy_spend_key))
- )
- if local_spend >= max_budget or self.dual_cache.redis_cache is not None:
+ current_spend: Final = _as_spend(await self._cached_spend(spend_key))
+ if legacy_spend_key is None or legacy_spend_key == spend_key:
+ return current_spend
+ return current_spend + _as_spend(await self._cached_spend(legacy_spend_key))
+
+ async def _with_shared_spend(
+ self,
+ entity_type: Litellm_EntityType,
+ entity_id: str | None,
+ resolved: ResolvedModelBudget,
+ local_spend: float,
+ ) -> float:
+ """Without Redis each replica's counter holds only its own share, so the window every replica adds to
+ in the database is the floor."""
+ if self.dual_cache.redis_cache is not None:
return local_spend
shared: Final = await _open_shared_window(
shared_windows=self._shared_windows,
- spend_key=spend_key,
+ spend_key=model_budget_spend_cache_key(
+ entity_type=entity_type,
+ entity_id=entity_id,
+ budget_model=resolved.budget_model,
+ budget_duration=resolved.budget_config.budget_duration,
+ ),
budget_duration=resolved.budget_config.budget_duration,
now=self._clock(),
)
@@ -878,6 +912,9 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
budget_model=resolved.budget_model,
budget_duration=budget_duration,
)
+ await self._join_shared_window(
+ spend_key=spend_key, start_time_key=start_time_key, budget_duration=budget_duration
+ )
await self._increment_spend_for_key(
budget_config=resolved.budget_config,
spend_key=spend_key,
@@ -891,32 +928,35 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
response_cost=response_cost,
)
- async def _share_spend(
- self, spend_key: str, start_time_key: str, budget_duration: str, response_cost: float
- ) -> None:
- """Without Redis, add this charge to the window every replica reads.
+ async def _join_shared_window(self, spend_key: str, start_time_key: str, budget_duration: str) -> None:
+ """Without Redis, start this replica's window where the database says every replica's window starts.
- A replica joins the window that is already open rather than its own, so
- all of them roll over together and none resets the others' spend.
+ Every replica then sends the same window start with its spend, so no flush reads as a newer window that
+ replaces the spend the others already added.
"""
if self.dual_cache.redis_cache is not None:
return
now: Final = self._clock()
- shared: Final = await _open_shared_window(
- shared_windows=self._shared_windows,
+ local: Final = self.dual_cache.in_memory_cache
+ if _open_local_window_start(
+ local=local, start_time_key=start_time_key, budget_duration=budget_duration, now=now
+ ):
+ return
+ _start_local_window(
+ local=local,
spend_key=spend_key,
+ start_time_key=start_time_key,
+ started_at=await self._shared_windows.open(spend_key=spend_key, budget_duration=budget_duration, now=now),
budget_duration=budget_duration,
now=now,
)
- if shared is not None:
- _join_window(
- local=self.dual_cache.in_memory_cache,
- spend_key=spend_key,
- start_time_key=start_time_key,
- window=shared,
- budget_duration=budget_duration,
- now=now,
- )
+
+ async def _share_spend(
+ self, spend_key: str, start_time_key: str, budget_duration: str, response_cost: float
+ ) -> None:
+ if self.dual_cache.redis_cache is not None:
+ return
+ now: Final = self._clock()
await self._shared_windows.add(
spend_key=spend_key,
budget_duration=budget_duration,
@@ -936,14 +976,17 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
return polls == 1
-def _join_window(
- local: LocalTTLCache, spend_key: str, start_time_key: str, window: SharedWindow, budget_duration: str, now: float
+def _open_local_window_start(local: LocalTTLCache, start_time_key: str, budget_duration: str, now: float) -> bool:
+ started_at: Final = _as_float(local.get_cache(key=start_time_key))
+ return started_at is not None and now - started_at <= budget_window_seconds(started_at, budget_duration)
+
+
+def _start_local_window(
+ local: LocalTTLCache, spend_key: str, start_time_key: str, started_at: float, budget_duration: str, now: float
) -> None:
- remaining: Final = window.started_at + budget_window_seconds(window.started_at, budget_duration) - now
- local.set_cache(key=start_time_key, value=window.started_at, ttl=remaining)
- counted: Final = local.get_cache(key=spend_key)
- if counted is not None:
- local.set_cache(key=spend_key, value=counted, ttl=remaining)
+ ttl: Final = started_at + budget_window_seconds(started_at, budget_duration) - now
+ local.set_cache(key=spend_key, value=0.0, ttl=ttl)
+ local.set_cache(key=start_time_key, value=started_at, ttl=ttl)
PROXY_VirtualKeyModelMaxBudgetLimiter = _PROXY_VirtualKeyModelMaxBudgetLimiter
diff --git a/tests/unit/proxy/db/test_budget_window_spend_writer.py b/tests/unit/proxy/db/test_budget_window_spend_writer.py
index 6fc438feee8..27f1472c183 100644
--- a/tests/unit/proxy/db/test_budget_window_spend_writer.py
+++ b/tests/unit/proxy/db/test_budget_window_spend_writer.py
@@ -9,6 +9,7 @@ import pytest
from litellm.proxy.db.budget_window_spend_writer import (
WindowSeedTotals,
commit_window_spend_updates,
+ open_window_spend_row,
roll_window_spend_row,
spend_logs_seed_totals,
)
@@ -405,6 +406,29 @@ async def test_roll_window_spend_row_is_conditional_on_the_stored_window_being_o
assert params[:4] == ("team", "t1", "30d", datetime(2026, 8, 31))
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+ ("replaces_window_start", "replaced_param"), [(None, None), (WINDOW_A, datetime(2026, 8, 1))]
+)
+async def test_open_window_spend_row_moves_only_the_window_the_caller_saw_end(replaces_window_start, replaced_param):
+ db = _FakeDB()
+
+ await open_window_spend_row(
+ prisma_client=_FakePrismaClient(db),
+ entity_type="model_max_budget",
+ entity_id="k1",
+ window_duration="1d",
+ window_start=WINDOW_B,
+ replaces_window_start=replaces_window_start,
+ )
+
+ ((query, params),) = db.execute_raw_calls
+ normalized = " ".join(query.split())
+ assert "ON CONFLICT (entity_type, entity_id, window_duration) DO UPDATE" in normalized
+ assert normalized.endswith("WHERE \"LiteLLM_BudgetWindowSpend\".window_start <= ($5::timestamptz AT TIME ZONE 'UTC')")
+ assert params == ("model_max_budget", "k1", "1d", datetime(2026, 8, 31), replaced_param)
+
+
@pytest.mark.asyncio
async def test_seed_receives_the_batch_earliest_start_as_its_cutoff():
db = _FakeDB(existing_rows=[])
diff --git a/tests/unit/proxy/hooks/test_model_budget_shared_windows.py b/tests/unit/proxy/hooks/test_model_budget_shared_windows.py
index 70957b1f74d..db334119b2a 100644
--- a/tests/unit/proxy/hooks/test_model_budget_shared_windows.py
+++ b/tests/unit/proxy/hooks/test_model_budget_shared_windows.py
@@ -1,4 +1,4 @@
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
from types import SimpleNamespace
from typing import Final
@@ -24,6 +24,17 @@ class _Table:
return self.row
+class _Database:
+ def __init__(self, table: _Table) -> None:
+ self.litellm_budgetwindowspend = table
+ self.opened_with: tuple[object, ...] = ()
+
+ async def execute_raw(self, query: str, *params: object) -> int:
+ self.opened_with = params
+ self.litellm_budgetwindowspend.row = SimpleNamespace(window_start=params[3], spend=0.0)
+ return 1
+
+
class _Sink:
def __init__(self) -> None:
self.transactions: tuple[WindowSpendTransaction, ...] = ()
@@ -33,7 +44,7 @@ class _Sink:
def _database_windows(table: _Table | None, sink: _Sink) -> DatabaseSharedWindows:
- prisma: Final = None if table is None else SimpleNamespace(db=SimpleNamespace(litellm_budgetwindowspend=table))
+ prisma: Final = None if table is None else SimpleNamespace(db=_Database(table))
return DatabaseSharedWindows(
prisma_client_source=lambda: prisma, # pyright: ignore[reportArgumentType] # duck-typed prisma double
sink=sink,
@@ -91,3 +102,42 @@ async def test_nothing_is_queued_without_a_database():
assert sink.transactions == ()
assert await windows.read(spend_key=SPEND_KEY, budget_duration="1d") is None
+
+
+@pytest.mark.asyncio
+async def test_open_joins_the_window_another_replica_already_started():
+ table: Final = _Table(row=SimpleNamespace(window_start=WINDOW_START.replace(tzinfo=None), spend=2.5))
+ windows: Final = _database_windows(table, _Sink())
+
+ started_at: Final = await windows.open(
+ spend_key=SPEND_KEY, budget_duration="1d", now=WINDOW_START.timestamp() + 3600
+ )
+
+ assert started_at == WINDOW_START.timestamp()
+ assert table.row.spend == 2.5
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("stored_start", [None, WINDOW_START - timedelta(days=2)])
+async def test_open_starts_a_window_at_now_when_none_is_open(stored_start: datetime | None):
+ table: Final = _Table(
+ row=None if stored_start is None else SimpleNamespace(window_start=stored_start.replace(tzinfo=None), spend=9.0)
+ )
+ windows: Final = _database_windows(table, _Sink())
+
+ started_at: Final = await windows.open(spend_key=SPEND_KEY, budget_duration="1d", now=WINDOW_START.timestamp())
+
+ assert started_at == WINDOW_START.timestamp()
+ assert await windows.read(spend_key=SPEND_KEY, budget_duration="1d") == SharedWindow(
+ started_at=WINDOW_START.timestamp(), spend=0.0
+ )
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("table", [None, _Table(error=RuntimeError("database is down"))])
+async def test_open_falls_back_to_this_replicas_own_window_without_a_database(table: _Table | None):
+ windows: Final = _database_windows(table, _Sink())
+
+ assert await windows.open(spend_key=SPEND_KEY, budget_duration="1d", now=WINDOW_START.timestamp()) == (
+ WINDOW_START.timestamp()
+ )
diff --git a/tests/unit/proxy/hooks/test_model_max_budget_limiter.py b/tests/unit/proxy/hooks/test_model_max_budget_limiter.py
index 33fa7bc3daf..73328f0616f 100644
--- a/tests/unit/proxy/hooks/test_model_max_budget_limiter.py
+++ b/tests/unit/proxy/hooks/test_model_max_budget_limiter.py
@@ -1,4 +1,6 @@
import asyncio
+import functools
+import re
import time
from collections.abc import Callable, Mapping
from types import MappingProxyType
@@ -333,18 +335,37 @@ def _model_access_groups(groups_by_model: Mapping[str, frozenset[str]]) -> Calla
class _SharedWindowsDouble:
def __init__(self) -> None:
self.rows: Mapping[str, SharedWindow] = MappingProxyType({})
+ self.queued: tuple[tuple[str, float, float], ...] = ()
async def read(self, spend_key: str, budget_duration: str) -> SharedWindow | None:
return self.rows.get(spend_key)
- async def add(self, spend_key: str, budget_duration: str, started_at: float, spend: float) -> None:
+ async def open(self, spend_key: str, budget_duration: str, now: float) -> float:
stored: Final = self.rows.get(spend_key)
- row: Final = (
- SharedWindow(started_at=stored.started_at, spend=stored.spend + spend)
- if stored is not None and stored.started_at >= started_at
- else SharedWindow(started_at=started_at, spend=spend)
- )
- self.rows = MappingProxyType({**self.rows, spend_key: row})
+ if stored is not None and stored.is_open(budget_duration=budget_duration, now=now):
+ return stored.started_at
+ self.rows = MappingProxyType({**self.rows, spend_key: SharedWindow(started_at=now, spend=0.0)})
+ return now
+
+ async def add(self, spend_key: str, budget_duration: str, started_at: float, spend: float) -> None:
+ self.queued = (*self.queued, (spend_key, started_at, spend))
+
+ def flush(self) -> None:
+ self.rows = functools.reduce(_upsert_window_spend, self.queued, self.rows)
+ self.queued = ()
+
+
+def _upsert_window_spend(
+ rows: Mapping[str, SharedWindow], queued: tuple[str, float, float]
+) -> Mapping[str, SharedWindow]:
+ spend_key, started_at, spend = queued
+ stored: Final = rows.get(spend_key)
+ row: Final = (
+ SharedWindow(started_at=stored.started_at, spend=stored.spend + spend)
+ if stored is not None and stored.started_at >= started_at
+ else SharedWindow(started_at=started_at, spend=spend)
+ )
+ return MappingProxyType({**rows, spend_key: row})
def _family_limiter(
@@ -393,7 +414,9 @@ async def test_every_opus_version_draws_from_one_family_entry(family_entry: str)
await _spend_on(limiter, model="claude-opus-4-8", model_max_budget=budget)
- with pytest.raises(litellm.BudgetExceededError, match="model=claude-opus-5-5"):
+ with pytest.raises(
+ litellm.BudgetExceededError, match=rf"model=claude-opus-5-5, budget_model={re.escape(family_entry)}$"
+ ):
await limiter.is_key_within_model_budget(user_api_key_dict=_key(budget), model="claude-opus-5-5")
assert await limiter.is_key_within_model_budget(user_api_key_dict=_key(budget), model="claude-sonnet-5")
@@ -488,6 +511,7 @@ async def test_replicas_without_redis_enforce_one_family_cap_together():
replica_b: Final = _family_limiter(shared_windows=table)
await _spend_on(replica_a, model="claude-opus-4-8", model_max_budget=budget)
+ table.flush()
with pytest.raises(litellm.BudgetExceededError, match="model=claude-opus-5-5"):
await replica_b.is_key_within_model_budget(user_api_key_dict=_key(budget), model="claude-opus-5-5")
@@ -509,6 +533,7 @@ async def test_replicas_sharing_redis_enforce_the_family_cap_without_the_databas
await _spend_on(replica_a, model="claude-opus-4-8", model_max_budget=budget)
await _drain_redis_pushes()
+ table.flush()
with pytest.raises(litellm.BudgetExceededError, match="model=claude-opus-5-5"):
await replica_b.is_key_within_model_budget(user_api_key_dict=_key(budget), model="claude-opus-5-5")
@@ -525,8 +550,10 @@ async def test_a_replica_charges_the_open_shared_window_instead_of_restarting_it
replica_b: Final = _family_limiter(shared_windows=table, clock=clock.now)
await _spend_on(replica_a, model="claude-opus-4-8", model_max_budget=budget)
+ table.flush()
clock.advance(3600)
await _spend_on(replica_b, model="claude-opus-5-5", model_max_budget=budget)
+ table.flush()
for replica in (replica_a, replica_b):
with pytest.raises(litellm.BudgetExceededError):
@@ -542,6 +569,7 @@ async def test_key_info_on_one_replica_reports_spend_from_every_replica():
await _spend_on(replica_a, model="claude-opus-4-8", model_max_budget=budget)
await _spend_on(replica_b, model="claude-opus-5-5", model_max_budget=budget)
+ table.flush()
usage: Final = await build_model_max_budget_usage(
entity_type=Litellm_EntityType.KEY,
@@ -552,3 +580,28 @@ async def test_key_info_on_one_replica_reports_spend_from_every_replica():
)
assert usage["claude-opus*"]["current_spend"] == round(CHAT_COST * 2, 4)
assert usage["claude-opus*"]["budget_reset_at"] is not None
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("budget_model", ["claude-opus*", "claude-opus-4-8"])
+async def test_replicas_starting_a_window_moments_apart_keep_each_others_spend(budget_model: str):
+ budget: Final = {budget_model: ROOMY_CAP}
+ table: Final = _SharedWindowsDouble()
+ clock: Final = _Clock()
+ clock.advance(time.time())
+ replica_a: Final = _family_limiter(shared_windows=table, clock=clock.now)
+ replica_b: Final = _family_limiter(shared_windows=table, clock=clock.now)
+
+ await _spend_on(replica_a, model="claude-opus-4-8", model_max_budget=budget)
+ clock.advance(1)
+ await _spend_on(replica_b, model="claude-opus-4-8", model_max_budget=budget)
+ table.flush()
+
+ usage: Final = await build_model_max_budget_usage(
+ entity_type=Litellm_EntityType.KEY,
+ entity_id=KEY_HASH,
+ model_max_budget=budget,
+ cache=DualCache(),
+ shared_windows=table,
+ )
+ assert usage[budget_model]["current_spend"] == round(CHAT_COST * 2, 4)
diff --git a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py
index d6ee9b1d29e..fdc40f05146 100644
--- a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py
+++ b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py
@@ -16582,6 +16582,7 @@ async def test_info_key_fn_includes_model_max_budget_usage(monkeypatch):
}
mock_prisma_client = AsyncMock()
+ mock_prisma_client.db.litellm_budgetwindowspend.find_unique = AsyncMock(return_value=None)
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client)
mock_user_api_key_cache = AsyncMock()
monkeypatch.setattr(
@@ -16705,6 +16706,7 @@ async def test_info_key_fn_v2_includes_model_max_budget_usage(monkeypatch):
model_max_budget = {"gpt-4o": {"budget_limit": 1.00, "time_period": "7d"}}
mock_prisma_client = AsyncMock()
+ mock_prisma_client.db.litellm_budgetwindowspend.find_unique = AsyncMock(return_value=None)
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client)
mock_user_api_key_cache = AsyncMock()
monkeypatch.setattr(
@@ -16769,6 +16771,7 @@ async def test_info_key_fn_budget_table_fallback(monkeypatch):
}
mock_prisma_client = AsyncMock()
+ mock_prisma_client.db.litellm_budgetwindowspend.find_unique = AsyncMock(return_value=None)
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client)
mock_user_api_key_cache = AsyncMock()
monkeypatch.setattr(
@@ -16842,6 +16845,7 @@ async def test_info_key_fn_v2_budget_table_fallback(monkeypatch):
}
mock_prisma_client = AsyncMock()
+ mock_prisma_client.db.litellm_budgetwindowspend.find_unique = AsyncMock(return_value=None)
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client)
mock_user_api_key_cache = AsyncMock()
monkeypatch.setattr(
@@ -17217,6 +17221,7 @@ async def test_info_key_fn_reads_the_configured_budget_model_key(monkeypatch):
}
mock_prisma_client = AsyncMock()
+ mock_prisma_client.db.litellm_budgetwindowspend.find_unique = AsyncMock(return_value=None)
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client)
mock_user_api_key_cache = AsyncMock()
monkeypatch.setattr(
diff --git a/tests/unit/test_internal_context.py b/tests/unit/test_internal_context.py
index 60a75c688c4..713d91e3016 100644
--- a/tests/unit/test_internal_context.py
+++ b/tests/unit/test_internal_context.py
@@ -74,6 +74,7 @@ _IN_MEMORY_ONLY_CALLERS: Final = frozenset(
"litellm/proxy/container_endpoints/ownership.py",
"litellm/proxy/discovery_endpoints/agent_skills_endpoints.py",
"litellm/proxy/guardrails/guardrail_hooks/straiker/straiker.py",
+ "litellm/proxy/hooks/model_budget_shared_windows.py",
"litellm/proxy/spend_tracking/key_metadata_recovery.py",
"litellm/proxy/ui_crud_endpoints/latest_release_endpoints.py",
"litellm/responses/litellm_completion_transformation/transformation.py",
diff --git a/ui/litellm-dashboard/src/components/key_team_helpers/ModelMaxBudgetEditor.integration.test.tsx b/ui/litellm-dashboard/src/components/key_team_helpers/ModelMaxBudgetEditor.integration.test.tsx
index c8fb108f66d..f7f0acd3254 100644
--- a/ui/litellm-dashboard/src/components/key_team_helpers/ModelMaxBudgetEditor.integration.test.tsx
+++ b/ui/litellm-dashboard/src/components/key_team_helpers/ModelMaxBudgetEditor.integration.test.tsx
@@ -83,11 +83,32 @@ describe("ModelMaxBudgetEditor model families", () => {
);
await user.click(addButton());
- const modelField = screen.getByPlaceholderText("Select a model or type a pattern");
+ const modelField = screen.getByPlaceholderText("Select model");
await user.type(modelField, "claude-opus*");
await user.click(await screen.findByRole("option", { name: /claude-opus\*/ }));
fireEvent.change(screen.getByPlaceholderText("Max spend ($)"), { target: { value: "10" } });
expect(onChange).toHaveBeenLastCalledWith({ "claude-opus*": { budget_limit: 10, time_period: "30d" } });
});
+
+ it("does not offer a typed pattern that another row already budgets", async () => {
+ const user = userEvent.setup();
+ renderWithProviders(
+ ,
+ );
+
+ await user.click(addButton());
+ const [, newRowModelField] = screen.getAllByPlaceholderText("Select model");
+ await user.type(newRowModelField, "claude-opus*");
+
+ expect(screen.queryByRole("option", { name: /claude-opus\*/ })).not.toBeInTheDocument();
+ await user.clear(newRowModelField);
+ await user.type(newRowModelField, "claude-sonnet*");
+ expect(await screen.findByRole("option", { name: /claude-sonnet\*/ })).toBeInTheDocument();
+ });
});
diff --git a/ui/litellm-dashboard/src/components/key_team_helpers/ModelMaxBudgetEditor.tsx b/ui/litellm-dashboard/src/components/key_team_helpers/ModelMaxBudgetEditor.tsx
index 0fc0f2116be..304d0a6ad1d 100644
--- a/ui/litellm-dashboard/src/components/key_team_helpers/ModelMaxBudgetEditor.tsx
+++ b/ui/litellm-dashboard/src/components/key_team_helpers/ModelMaxBudgetEditor.tsx
@@ -163,9 +163,10 @@ export function ModelMaxBudgetEditor({
options={modelOptions.map((model) => ({ label: model, value: model }))}
value={entry.model}
onValueChange={(model) => updateEntry(entry.id, { model })}
- placeholder="Select a model or type a pattern"
+ placeholder="Select model"
emptyText="No models found"
customValueHint="Access group or model pattern"
+ reservedValues={entries.flatMap((other) => (other.id !== entry.id && other.model ? [other.model] : []))}
disabled={!premiumUser}
/>
diff --git a/ui/litellm-dashboard/src/components/shared/SearchSelect.tsx b/ui/litellm-dashboard/src/components/shared/SearchSelect.tsx
index 343194dc046..9b1203771d3 100644
--- a/ui/litellm-dashboard/src/components/shared/SearchSelect.tsx
+++ b/ui/litellm-dashboard/src/components/shared/SearchSelect.tsx
@@ -30,6 +30,7 @@ interface SearchSelectProps {
inputId?: string;
allowClear?: boolean;
customValueHint?: string;
+ reservedValues?: readonly string[];
"aria-label"?: string;
id?: string;
}
@@ -51,6 +52,7 @@ export function SearchSelect({
inputId,
allowClear = true,
customValueHint,
+ reservedValues = [],
"aria-label": ariaLabel,
id,
}: SearchSelectProps) {
@@ -60,8 +62,9 @@ export function SearchSelect({
const known =
selected !== null && !options.some((option) => option.value === selected.value) ? [selected, ...options] : options;
const typed = query.trim();
+ const isNewValue = typed !== "" && !reservedValues.includes(typed) && !known.some((option) => option.value === typed);
const items =
- customValueHint !== undefined && typed !== "" && !known.some((option) => option.value === typed)
+ customValueHint !== undefined && isNewValue
? [...known, { label: typed, value: typed, sublabel: customValueHint }]
: known;
From afcfa6a35b6de46cc4323c1d6607663db3309661 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 20 Oct 2026 21:03:20 -0700
Subject: [PATCH 4/6] fix(proxy): read the shared budget window from the
writer, and keep concurrent first charges on one replica
Opening a shared window now reads the row from the writer, since a lagging read replica could hand a replica its
own start and let a flush replace the others' spend. A charge that waited on the open no longer resets a local
window another charge on the same replica opened meanwhile. Window tests use fixed timestamps.
---
.../hooks/model_budget_shared_windows.py | 14 ++++---
.../proxy/hooks/model_max_budget_limiter.py | 9 ++++-
.../hooks/test_model_budget_shared_windows.py | 30 +++++++++++++-
.../hooks/test_model_max_budget_limiter.py | 40 ++++++++++++++++---
4 files changed, 79 insertions(+), 14 deletions(-)
diff --git a/litellm/proxy/hooks/model_budget_shared_windows.py b/litellm/proxy/hooks/model_budget_shared_windows.py
index 56729a33034..21f44ae6d6d 100644
--- a/litellm/proxy/hooks/model_budget_shared_windows.py
+++ b/litellm/proxy/hooks/model_budget_shared_windows.py
@@ -96,7 +96,7 @@ class DatabaseSharedWindows:
return now
try:
seen: Final = await _fetch_row(
- prisma_client=prisma_client, spend_key=spend_key, budget_duration=budget_duration
+ prisma_client=prisma_client, spend_key=spend_key, budget_duration=budget_duration, use_writer=True
)
if seen is not None and seen.is_open(budget_duration=budget_duration, now=now):
return seen.started_at
@@ -111,7 +111,7 @@ class DatabaseSharedWindows:
else datetime.fromtimestamp(seen.started_at, tz=timezone.utc),
)
opened: Final = await _fetch_row(
- prisma_client=prisma_client, spend_key=spend_key, budget_duration=budget_duration
+ prisma_client=prisma_client, spend_key=spend_key, budget_duration=budget_duration, use_writer=True
)
except Exception: # noqa: BLE001 # an unreachable database leaves this replica on its own window
verbose_proxy_logger.exception("Unable to open the shared model budget window")
@@ -141,14 +141,18 @@ class DatabaseSharedWindows:
if prisma_client is None:
return None
try:
- return await _fetch_row(prisma_client=prisma_client, spend_key=spend_key, budget_duration=budget_duration)
+ return await _fetch_row(
+ prisma_client=prisma_client, spend_key=spend_key, budget_duration=budget_duration, use_writer=False
+ )
except Exception: # noqa: BLE001 # an unreadable row leaves enforcement on this replica's own counter
verbose_proxy_logger.exception("Unable to read shared model budget spend")
return None
-async def _fetch_row(prisma_client: "PrismaClient", spend_key: str, budget_duration: str) -> SharedWindow | None:
- row: Final = await BudgetWindowSpendRepository(prisma_client).table.find_unique(
+async def _fetch_row(
+ prisma_client: "PrismaClient", spend_key: str, budget_duration: str, use_writer: bool
+) -> SharedWindow | None:
+ row: Final = await BudgetWindowSpendRepository(prisma_client, use_writer=use_writer).table.find_unique(
where={
"entity_type_entity_id_window_duration": {
"entity_type": MODEL_BUDGET_WINDOW_ENTITY_TYPE,
diff --git a/litellm/proxy/hooks/model_max_budget_limiter.py b/litellm/proxy/hooks/model_max_budget_limiter.py
index 1d7b7245d8a..b8ac75aec3a 100644
--- a/litellm/proxy/hooks/model_max_budget_limiter.py
+++ b/litellm/proxy/hooks/model_max_budget_limiter.py
@@ -938,6 +938,13 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
return
now: Final = self._clock()
local: Final = self.dual_cache.in_memory_cache
+ if _open_local_window_start(
+ local=local, start_time_key=start_time_key, budget_duration=budget_duration, now=now
+ ):
+ return
+ started_at: Final = await self._shared_windows.open(
+ spend_key=spend_key, budget_duration=budget_duration, now=now
+ )
if _open_local_window_start(
local=local, start_time_key=start_time_key, budget_duration=budget_duration, now=now
):
@@ -946,7 +953,7 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
local=local,
spend_key=spend_key,
start_time_key=start_time_key,
- started_at=await self._shared_windows.open(spend_key=spend_key, budget_duration=budget_duration, now=now),
+ started_at=started_at,
budget_duration=budget_duration,
now=now,
)
diff --git a/tests/unit/proxy/hooks/test_model_budget_shared_windows.py b/tests/unit/proxy/hooks/test_model_budget_shared_windows.py
index db334119b2a..8b103c33d32 100644
--- a/tests/unit/proxy/hooks/test_model_budget_shared_windows.py
+++ b/tests/unit/proxy/hooks/test_model_budget_shared_windows.py
@@ -43,8 +43,22 @@ class _Sink:
self.transactions = (*self.transactions, transaction)
-def _database_windows(table: _Table | None, sink: _Sink) -> DatabaseSharedWindows:
- prisma: Final = None if table is None else SimpleNamespace(db=_Database(table))
+class _LaggingReplica:
+ def __init__(self, writer: _Database) -> None:
+ self.litellm_budgetwindowspend = _Table()
+ self._writer = writer
+
+ async def execute_raw(self, query: str, *params: object) -> int:
+ return await self._writer.execute_raw(query, *params)
+
+
+def _database_windows(table: _Table | None, sink: _Sink, lagging_replica: bool = False) -> DatabaseSharedWindows:
+ writer: Final = None if table is None else _Database(table)
+ prisma: Final = (
+ None
+ if writer is None
+ else SimpleNamespace(db=_LaggingReplica(writer) if lagging_replica else writer, writer_db=writer)
+ )
return DatabaseSharedWindows(
prisma_client_source=lambda: prisma, # pyright: ignore[reportArgumentType] # duck-typed prisma double
sink=sink,
@@ -117,6 +131,18 @@ async def test_open_joins_the_window_another_replica_already_started():
assert table.row.spend == 2.5
+@pytest.mark.asyncio
+async def test_open_reads_the_window_from_the_writer_while_the_read_replica_lags():
+ table: Final = _Table(row=SimpleNamespace(window_start=WINDOW_START.replace(tzinfo=None), spend=2.5))
+ windows: Final = _database_windows(table, _Sink(), lagging_replica=True)
+
+ started_at: Final = await windows.open(
+ spend_key=SPEND_KEY, budget_duration="1d", now=WINDOW_START.timestamp() + 3600
+ )
+
+ assert started_at == WINDOW_START.timestamp()
+
+
@pytest.mark.asyncio
@pytest.mark.parametrize("stored_start", [None, WINDOW_START - timedelta(days=2)])
async def test_open_starts_a_window_at_now_when_none_is_open(stored_start: datetime | None):
diff --git a/tests/unit/proxy/hooks/test_model_max_budget_limiter.py b/tests/unit/proxy/hooks/test_model_max_budget_limiter.py
index 73328f0616f..cf925273163 100644
--- a/tests/unit/proxy/hooks/test_model_max_budget_limiter.py
+++ b/tests/unit/proxy/hooks/test_model_max_budget_limiter.py
@@ -3,6 +3,7 @@ import functools
import re
import time
from collections.abc import Callable, Mapping
+from datetime import datetime, timezone
from types import MappingProxyType
from typing import Final
@@ -325,7 +326,11 @@ OPUS_GROUP_MEMBERS: Final = MappingProxyType(
{"claude-opus-4-8": frozenset({OPUS_GROUP}), "claude-opus-5-5": frozenset({OPUS_GROUP})}
)
FAMILY_CAP: Final = {"budget_limit": CHAT_COST, "time_period": "1d"} # mutable-ok: BudgetConfig validates dicts only
-ROOMY_CAP: Final = {"budget_limit": CHAT_COST * 100, "time_period": "1d"} # mutable-ok: BudgetConfig validates dicts only
+ROOMY_CAP: Final = {
+ "budget_limit": CHAT_COST * 100,
+ "time_period": "1d",
+} # mutable-ok: BudgetConfig validates dicts only
+WINDOW_EPOCH: Final = datetime(2026, 10, 20, 9, 30, tzinfo=timezone.utc).timestamp()
def _model_access_groups(groups_by_model: Mapping[str, frozenset[str]]) -> Callable[[str], frozenset[str]]:
@@ -341,6 +346,7 @@ class _SharedWindowsDouble:
return self.rows.get(spend_key)
async def open(self, spend_key: str, budget_duration: str, now: float) -> float:
+ await asyncio.sleep(0)
stored: Final = self.rows.get(spend_key)
if stored is not None and stored.is_open(budget_duration=budget_duration, now=now):
return stored.started_at
@@ -493,9 +499,7 @@ async def test_key_info_usage_reports_the_family_total():
("*", "claude-sonnet-5", True),
],
)
-def test_glob_entries_match_the_whole_model_name_with_only_star_as_a_wildcard(
- pattern: str, model: str, governs: bool
-):
+def test_glob_entries_match_the_whole_model_name_with_only_star_as_a_wildcard(pattern: str, model: str, governs: bool):
resolved: Final = resolve_model_budget(
model=model, model_max_budget={pattern: FAMILY_CAP}, model_access_groups=_model_access_groups({})
)
@@ -545,7 +549,7 @@ async def test_a_replica_charges_the_open_shared_window_instead_of_restarting_it
budget: Final = {"claude-opus*": {"budget_limit": CHAT_COST * 1.5, "time_period": "1d"}}
table: Final = _SharedWindowsDouble()
clock: Final = _Clock()
- clock.advance(time.time())
+ clock.advance(WINDOW_EPOCH)
replica_a: Final = _family_limiter(shared_windows=table, clock=clock.now)
replica_b: Final = _family_limiter(shared_windows=table, clock=clock.now)
@@ -588,7 +592,7 @@ async def test_replicas_starting_a_window_moments_apart_keep_each_others_spend(b
budget: Final = {budget_model: ROOMY_CAP}
table: Final = _SharedWindowsDouble()
clock: Final = _Clock()
- clock.advance(time.time())
+ clock.advance(WINDOW_EPOCH)
replica_a: Final = _family_limiter(shared_windows=table, clock=clock.now)
replica_b: Final = _family_limiter(shared_windows=table, clock=clock.now)
@@ -602,6 +606,30 @@ async def test_replicas_starting_a_window_moments_apart_keep_each_others_spend(b
entity_id=KEY_HASH,
model_max_budget=budget,
cache=DualCache(),
+ now=datetime.fromtimestamp(clock.now(), tz=timezone.utc),
shared_windows=table,
)
assert usage[budget_model]["current_spend"] == round(CHAT_COST * 2, 4)
+
+
+@pytest.mark.asyncio
+async def test_concurrent_first_charges_on_one_replica_both_count():
+ budget: Final = {"claude-opus*": ROOMY_CAP}
+ clock: Final = _Clock()
+ clock.advance(WINDOW_EPOCH)
+ limiter: Final = _family_limiter(shared_windows=_SharedWindowsDouble(), clock=clock.now)
+
+ await asyncio.gather(
+ _spend_on(limiter, model="claude-opus-4-8", model_max_budget=budget),
+ _spend_on(limiter, model="claude-opus-5-5", model_max_budget=budget),
+ )
+
+ usage: Final = await build_model_max_budget_usage(
+ entity_type=Litellm_EntityType.KEY,
+ entity_id=KEY_HASH,
+ model_max_budget=budget,
+ cache=limiter.dual_cache,
+ now=datetime.fromtimestamp(clock.now(), tz=timezone.utc),
+ shared_windows=_SharedWindowsDouble(),
+ )
+ assert usage["claude-opus*"]["current_spend"] == round(CHAT_COST * 2, 4)
From 053580ffce5288f49d60e16c10ab7c87d4ebaa93 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 20 Oct 2026 22:13:44 -0700
Subject: [PATCH 5/6] fix(proxy): stop a non proxy admin from lifting a team
model cap with a looser group or pattern entry
Group and pattern entries now take precedence over other entries, so a team update by anyone but a proxy admin
must keep each new or changed entry within every existing cap it can take precedence over. A pattern can match a
provider-prefixed or wildcard-routed name, so any two patterns count as overlapping, and a model or group entry
counts as outranking every pattern.
---
.../proxy/hooks/model_max_budget_limiter.py | 53 ++++++++++
.../management_endpoints/team_endpoints.py | 77 ++++++++++++++-
.../test_team_endpoints.py | 98 +++++++++++++++++++
3 files changed, 223 insertions(+), 5 deletions(-)
diff --git a/litellm/proxy/hooks/model_max_budget_limiter.py b/litellm/proxy/hooks/model_max_budget_limiter.py
index b8ac75aec3a..d6f5bb1c5bf 100644
--- a/litellm/proxy/hooks/model_max_budget_limiter.py
+++ b/litellm/proxy/hooks/model_max_budget_limiter.py
@@ -55,6 +55,8 @@ _GLOB_WILDCARD: Final = "*"
ModelAccessGroups: TypeAlias = Callable[[str], Collection[str]]
+AccessGroupMembers: TypeAlias = Callable[[str], Collection[str]]
+
_BUDGET_START_TIME_KEY_PREFIXES: Final = MappingProxyType(
{
Litellm_EntityType.KEY: "virtual_key_budget_start_time",
@@ -153,6 +155,16 @@ def proxy_model_access_groups(model: str) -> frozenset[str]:
return frozenset(llm_router.get_model_access_groups(model_name=model))
+def proxy_access_group_members(access_group: str) -> frozenset[str]:
+ try:
+ from litellm.proxy.proxy_server import llm_router
+ except ImportError:
+ return frozenset()
+ if llm_router is None:
+ return frozenset()
+ return frozenset(llm_router.get_model_access_groups(model_access_group=access_group).get(access_group, ()))
+
+
def resolve_model_budget(
model: str,
model_max_budget: Mapping[str, object],
@@ -223,6 +235,47 @@ def _glob_precedence(pattern: str) -> tuple[tuple[int, int], str]:
return PatternUtils.calculate_pattern_specificity(pattern), pattern
+def entries_outranked_by(
+ entry: str,
+ model_max_budget: Collection[str],
+ access_group_members: AccessGroupMembers = proxy_access_group_members,
+) -> Iterator[str]:
+ """Entries that `entry` can take precedence over for some request model.
+
+ A pattern can match a provider-prefixed or wildcard-routed name whose
+ candidates or access groups include any other entry, so patterns are taken
+ to overlap: a pattern outranks every less specific pattern, and a model or
+ group entry outranks every pattern.
+ """
+ if _GLOB_WILDCARD in entry:
+ entry_precedence: Final = _glob_precedence(entry)
+ yield from (
+ other
+ for other in model_max_budget
+ if _GLOB_WILDCARD in other and _glob_precedence(other) < entry_precedence
+ )
+ return
+ yield from (other for other in model_max_budget if _GLOB_WILDCARD in other)
+ entry_members: Final = frozenset(access_group_members(entry))
+ yield from (
+ other
+ for other in model_max_budget
+ if _GLOB_WILDCARD not in other
+ and _ranks_ahead_of_group(
+ entry=entry, entry_members=entry_members, group=other, group_members=access_group_members(other)
+ )
+ )
+
+
+def _ranks_ahead_of_group(entry: str, entry_members: frozenset[str], group: str, group_members: Iterable[str]) -> bool:
+ return any(
+ _GLOB_WILDCARD in member
+ or entry in _budget_model_candidates(member)
+ or (member in entry_members and entry < group)
+ for member in group_members
+ )
+
+
def _usable_entry(entry: str, raw_budget_config: object) -> ResolvedModelBudget | None:
if (budget_config := _usable_budget_config(raw_budget_config)) is None:
# An entry that will not validate cannot be keyed, so it cannot be
diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py
index d9763f72741..c1b81a345d0 100644
--- a/litellm/proxy/management_endpoints/team_endpoints.py
+++ b/litellm/proxy/management_endpoints/team_endpoints.py
@@ -137,7 +137,10 @@ from litellm.proxy.common_utils.user_api_key_cache import (
from litellm.proxy.db.db_span import db_span
from litellm.proxy.hooks.key_management_event_hooks import KeyManagementEventHooks
from litellm.proxy.hooks.model_max_budget_limiter import (
+ AccessGroupMembers,
build_model_max_budget_usage,
+ entries_outranked_by,
+ proxy_access_group_members,
resolve_model_budget,
)
from litellm.proxy.management.teams.authz import TEAM_OR_ORG_ADMIN, TeamRole, is_team_admin, team_access_denied
@@ -1427,6 +1430,7 @@ def _check_team_model_budget_update_authority(
data: UpdateTeamRequest,
user_api_key_dict: UserAPIKeyAuth,
existing_model_max_budget: Mapping[str, object] | None,
+ access_group_members: AccessGroupMembers = proxy_access_group_members,
) -> None:
"""Like `_check_team_budget_update_authority`: only a proxy admin may raise, re-window or drop a per-model cap."""
if user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN:
@@ -1454,11 +1458,7 @@ def _check_team_model_budget_update_authority(
cap = governing.budget_config
if cap.max_budget is None:
continue
- if (
- proposed.max_budget is None
- or proposed.max_budget > cap.max_budget
- or proposed.budget_duration != cap.budget_duration
- ):
+ if _looser_model_cap(proposed=proposed, cap=cap):
raise HTTPException(
status_code=403,
detail={
@@ -1470,6 +1470,73 @@ def _check_team_model_budget_update_authority(
)
},
)
+ for model_name, proposed in requested.items():
+ _check_outranked_model_caps(
+ entry=model_name,
+ proposed=proposed,
+ existing_model_max_budget=existing_model_max_budget,
+ access_group_members=access_group_members,
+ )
+
+
+def _check_outranked_model_caps(
+ entry: str,
+ proposed: BudgetConfig,
+ existing_model_max_budget: Mapping[str, object],
+ access_group_members: AccessGroupMembers,
+) -> None:
+ if _keeps_or_tightens(proposed=proposed, existing=_existing_model_cap(existing_model_max_budget.get(entry))):
+ return
+ outranked: Final = next(
+ _looser_than_outranked_caps(entry, proposed, existing_model_max_budget, access_group_members), None
+ )
+ if outranked is None:
+ return
+ outranked_name, cap = outranked
+ raise HTTPException(
+ status_code=403,
+ detail={
+ "error": (
+ f"Only a proxy admin can set a team's model_max_budget entry {entry!r} looser than "
+ f"{outranked_name!r}, which it can take precedence over. Current max_budget={cap.max_budget} per "
+ f"{cap.budget_duration} (entry {outranked_name!r}), requested={proposed.max_budget} per "
+ f"{proposed.budget_duration}."
+ )
+ },
+ )
+
+
+def _looser_than_outranked_caps(
+ entry: str,
+ proposed: BudgetConfig,
+ existing_model_max_budget: Mapping[str, object],
+ access_group_members: AccessGroupMembers,
+) -> Iterator[tuple[str, BudgetConfig]]:
+ outranked_caps: Final = (
+ (outranked, _existing_model_cap(existing_model_max_budget[outranked]))
+ for outranked in entries_outranked_by(
+ entry=entry, model_max_budget=existing_model_max_budget, access_group_members=access_group_members
+ )
+ )
+ yield from (
+ (outranked, cap)
+ for outranked, cap in outranked_caps
+ if cap is not None and cap.max_budget is not None and _looser_model_cap(proposed=proposed, cap=cap)
+ )
+
+
+def _keeps_or_tightens(proposed: BudgetConfig, existing: BudgetConfig | None) -> bool:
+ return existing is not None and (
+ existing.max_budget is None or not _looser_model_cap(proposed=proposed, cap=existing)
+ )
+
+
+def _looser_model_cap(proposed: BudgetConfig, cap: BudgetConfig) -> bool:
+ return (
+ proposed.max_budget is None
+ or (cap.max_budget is not None and proposed.max_budget > cap.max_budget)
+ or proposed.budget_duration != cap.budget_duration
+ )
def _should_auto_add_team_creator(
diff --git a/tests/unit/proxy/management_endpoints/test_team_endpoints.py b/tests/unit/proxy/management_endpoints/test_team_endpoints.py
index 37028558af3..2d728b506e9 100644
--- a/tests/unit/proxy/management_endpoints/test_team_endpoints.py
+++ b/tests/unit/proxy/management_endpoints/test_team_endpoints.py
@@ -17119,6 +17119,104 @@ def test_team_admin_can_tighten_or_keep_team_model_caps(requested) -> None:
)
+_TEAM_OPUS_FAMILY_CAP: Final = MappingProxyType({"claude-opus*": {"max_budget": 10.0, "budget_duration": "1d"}})
+_TEAM_OPUS_GROUP_CAP: Final = MappingProxyType({"z-opus-group": {"max_budget": 10.0, "budget_duration": "1d"}})
+_TEAM_ACCESS_GROUP_MEMBERS: Final = MappingProxyType(
+ {
+ "z-opus-group": ("claude-opus-4-8",),
+ "a-opus-group": ("claude-opus-4-8", "claude-opus-5-5"),
+ "zz-opus-group": ("claude-opus-4-8",),
+ "a-gemini-group": ("gemini-3-pro",),
+ "z-anthropic-group": ("anthropic/*",),
+ }
+)
+
+
+def _team_access_group_members(access_group: str) -> tuple[str, ...]:
+ return _TEAM_ACCESS_GROUP_MEMBERS.get(access_group, ())
+
+
+def _check_team_admin_model_caps(existing: Mapping[str, object], requested: Mapping[str, object]) -> None:
+ from litellm.proxy.management_endpoints.team_endpoints import _check_team_model_budget_update_authority
+
+ _check_team_model_budget_update_authority(
+ data=UpdateTeamRequest(team_id="t1", model_max_budget=dict(requested)),
+ user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, user_id="team-admin"),
+ existing_model_max_budget=existing,
+ access_group_members=_team_access_group_members,
+ )
+
+
+@pytest.mark.parametrize(
+ ("existing", "added"),
+ [
+ (_TEAM_OPUS_FAMILY_CAP, {"opus-group": {"max_budget": 1000.0, "budget_duration": "1d"}}),
+ (_TEAM_OPUS_FAMILY_CAP, {"opus-group": {"max_budget": 10.0, "budget_duration": "7d"}}),
+ (_TEAM_OPUS_FAMILY_CAP, {"*laude-opus-4-8": {"max_budget": 1000.0, "budget_duration": "1d"}}),
+ (_TEAM_OPUS_GROUP_CAP, {"a-opus-group": {"max_budget": 1000.0, "budget_duration": "1d"}}),
+ (_TEAM_OPUS_GROUP_CAP, {"claude-opus-4-8": {"max_budget": 1000.0, "budget_duration": "1d"}}),
+ (
+ {"z-anthropic-group": {"max_budget": 10.0, "budget_duration": "1d"}},
+ {"claude-opus-4-8": {"max_budget": 1000.0, "budget_duration": "1d"}},
+ ),
+ ],
+ ids=[
+ "group_over_pattern",
+ "group_rewindows_pattern",
+ "more_specific_pattern",
+ "earlier_group_sharing_a_model",
+ "model_inside_group",
+ "model_under_wildcard_group",
+ ],
+)
+def test_team_admin_cannot_add_an_entry_looser_than_a_cap_it_outranks(
+ existing: Mapping[str, object], added: Mapping[str, object]
+) -> None:
+ with pytest.raises(HTTPException) as exc:
+ _check_team_admin_model_caps(existing=existing, requested={**existing, **added})
+ assert exc.value.status_code == 403
+ assert "proxy admin" in exc.value.detail["error"]
+ assert repr(next(iter(existing))) in exc.value.detail["error"]
+
+
+@pytest.mark.parametrize(
+ ("existing", "requested"),
+ [
+ (_TEAM_OPUS_FAMILY_CAP, {**_TEAM_OPUS_FAMILY_CAP, "opus-group": {"max_budget": 5.0, "budget_duration": "1d"}}),
+ (_TEAM_OPUS_FAMILY_CAP, {**_TEAM_OPUS_FAMILY_CAP, "claude*": {"max_budget": 1000.0, "budget_duration": "1d"}}),
+ (_TEAM_OPUS_GROUP_CAP, {**_TEAM_OPUS_GROUP_CAP, "a-gemini-group": {"max_budget": 1000.0, "budget_duration": "1d"}}),
+ (_TEAM_OPUS_GROUP_CAP, {**_TEAM_OPUS_GROUP_CAP, "zz-opus-group": {"max_budget": 1000.0, "budget_duration": "1d"}}),
+ (_TEAM_OPUS_GROUP_CAP, {**_TEAM_OPUS_GROUP_CAP, "gpt-4o": {"max_budget": 1000.0, "budget_duration": "1d"}}),
+ (
+ {**_TEAM_OPUS_FAMILY_CAP, "opus-group": {"max_budget": 100.0, "budget_duration": "1d"}},
+ {**_TEAM_OPUS_FAMILY_CAP, "opus-group": {"max_budget": 100.0, "budget_duration": "1d"}},
+ ),
+ (
+ {**_TEAM_OPUS_FAMILY_CAP, "opus-group": {"max_budget": 100.0, "budget_duration": "1d"}},
+ {**_TEAM_OPUS_FAMILY_CAP, "opus-group": {"max_budget": 50.0, "budget_duration": "1d"}},
+ ),
+ (
+ {**_TEAM_OPUS_FAMILY_CAP, "opus-group": {"rpm_limit": 10}},
+ {**_TEAM_OPUS_FAMILY_CAP, "opus-group": {"rpm_limit": 20}},
+ ),
+ ],
+ ids=[
+ "tighter_group",
+ "less_specific_pattern",
+ "group_without_shared_models",
+ "later_group_sharing_a_model",
+ "model_outside_group",
+ "unchanged_looser_entry",
+ "tightened_looser_entry",
+ "uncapped_entry_edited",
+ ],
+)
+def test_team_admin_can_add_entries_that_never_lift_an_outranked_cap(
+ existing: Mapping[str, object], requested: Mapping[str, object]
+) -> None:
+ assert _check_team_admin_model_caps(existing=existing, requested=requested) is None
+
+
def test_team_model_cap_authority_skips_omitted_field_malformed_rows_and_proxy_admins() -> None:
from litellm.proxy.management_endpoints.team_endpoints import _check_team_model_budget_update_authority
From 3d78d57662dc6fc1dba53194e68d8831ae8cb3a3 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Wed, 21 Oct 2026 01:45:06 -0700
Subject: [PATCH 6/6] fix(proxy): narrow model budget pattern overlap, keep
team caps under key patterns, and accept only typed patterns in the UI
---
.../proxy/hooks/model_max_budget_limiter.py | 190 ++++++++++++------
.../management_endpoints/team_endpoints.py | 17 +-
.../hooks/test_model_budget_shared_windows.py | 24 +++
.../hooks/test_model_max_budget_limiter.py | 86 ++++++++
.../test_team_endpoints.py | 53 +++--
.../ModelMaxBudgetEditor.integration.test.tsx | 18 ++
.../key_team_helpers/ModelMaxBudgetEditor.tsx | 4 +-
.../shared/SearchSelect.integration.test.tsx | 7 +-
.../src/components/shared/SearchSelect.tsx | 13 +-
9 files changed, 325 insertions(+), 87 deletions(-)
diff --git a/litellm/proxy/hooks/model_max_budget_limiter.py b/litellm/proxy/hooks/model_max_budget_limiter.py
index d6f5bb1c5bf..0531a079dea 100644
--- a/litellm/proxy/hooks/model_max_budget_limiter.py
+++ b/litellm/proxy/hooks/model_max_budget_limiter.py
@@ -1,12 +1,18 @@
import asyncio
import functools
+import itertools
import json
import time
from collections.abc import Callable, Collection, Iterable, Iterator, Mapping, Sequence
from dataclasses import dataclass
from datetime import datetime, timezone
from types import MappingProxyType
-from typing import Final, TypeAlias, cast # noqa: TID251 # untyped callback payload requires explicit narrowing
+from typing import (
+ TYPE_CHECKING,
+ Final,
+ TypeAlias,
+ cast, # noqa: TID251 # untyped callback payload requires explicit narrowing
+)
import litellm
from litellm._internal_context import with_service_target
@@ -25,9 +31,11 @@ from litellm.proxy.hooks.model_budget_shared_windows import (
)
from litellm.router_strategy.budget_limiter import RouterBudgetLimiting, budget_window_seconds
from litellm.router_utils.batch_utils import batch_charged_once_marker_key, batch_id_to_charge_once
-from litellm.router_utils.pattern_match_deployments import PatternUtils
from litellm.types.llms.openai import AllMessageValues
-from litellm.types.utils import BudgetConfig, StandardLoggingPayload
+from litellm.types.utils import BudgetConfig, LlmProviders, StandardLoggingPayload
+
+if TYPE_CHECKING:
+ from litellm.router import Router
VIRTUAL_KEY_SPEND_CACHE_KEY_PREFIX: Final = "virtual_key_spend"
END_USER_SPEND_CACHE_KEY_PREFIX: Final = "end_user_model_spend"
@@ -47,6 +55,8 @@ _SPEND_CACHE_KEY_PREFIXES: Final = MappingProxyType(
_LEGACY_REQUEST_MODEL_SCOPES: Final = frozenset({Litellm_EntityType.KEY, Litellm_EntityType.END_USER})
+_PER_REPLICA_SCOPES: Final = frozenset({Litellm_EntityType.END_USER})
+
_PROCESS_STARTED_AT: Final = time.monotonic()
_LATEST_RESET_AT: Final = datetime(9999, 12, 31, tzinfo=timezone.utc).timestamp()
@@ -55,7 +65,7 @@ _GLOB_WILDCARD: Final = "*"
ModelAccessGroups: TypeAlias = Callable[[str], Collection[str]]
-AccessGroupMembers: TypeAlias = Callable[[str], Collection[str]]
+_PROVIDER_PREFIXES: Final = tuple(f"{provider.value}/" for provider in LlmProviders)
_BUDGET_START_TIME_KEY_PREFIXES: Final = MappingProxyType(
{
@@ -82,6 +92,15 @@ class ResolvedModelBudget:
budget_config: BudgetConfig
+@dataclass(frozen=True, slots=True)
+class RoutedModels:
+ served_names: frozenset[str]
+ group_members: Mapping[str, frozenset[str]]
+
+ def members(self, access_group: str) -> frozenset[str]:
+ return self.group_members.get(access_group, frozenset())
+
+
def team_member_budget_entity_id(user_id: str, team_id: str) -> str:
return f"{len(user_id)}:{user_id}:{team_id}"
@@ -145,24 +164,48 @@ def model_budget_start_time_cache_key(
return f"{_BUDGET_START_TIME_KEY_PREFIXES[entity_type]}:{entity_id}:{budget_model}:{budget_duration}"
+def model_access_groups_of(router: "Router", model: str) -> frozenset[str]:
+ """Access groups serving `model`, leaving out any named after a model the router serves.
+
+ An entry with that name already budgets that model, so reading it as the group too would quietly widen it.
+ """
+ return frozenset(
+ group for group in router.get_model_access_groups(model_name=model) if group not in router.model_names
+ )
+
+
+def routed_models_of(router: "Router") -> RoutedModels:
+ served_names: Final = frozenset(router.model_names)
+ return RoutedModels(
+ served_names=served_names,
+ group_members=MappingProxyType(
+ {
+ group: frozenset(members)
+ for group, members in router.get_model_access_groups().items()
+ if group not in served_names
+ }
+ ),
+ )
+
+
def proxy_model_access_groups(model: str) -> frozenset[str]:
+ router: Final = _proxy_router()
+ return frozenset() if router is None else model_access_groups_of(router=router, model=model)
+
+
+def proxy_routed_models() -> RoutedModels:
+ router: Final = _proxy_router()
+ if router is None:
+ return RoutedModels(served_names=frozenset(), group_members=MappingProxyType({}))
+ return routed_models_of(router)
+
+
+def _proxy_router() -> "Router | None":
try:
from litellm.proxy.proxy_server import llm_router
except ImportError:
- return frozenset()
- if llm_router is None:
- return frozenset()
- return frozenset(llm_router.get_model_access_groups(model_name=model))
-
-
-def proxy_access_group_members(access_group: str) -> frozenset[str]:
- try:
- from litellm.proxy.proxy_server import llm_router
- except ImportError:
- return frozenset()
- if llm_router is None:
- return frozenset()
- return frozenset(llm_router.get_model_access_groups(model_access_group=access_group).get(access_group, ()))
+ return None
+ return llm_router
def resolve_model_budget(
@@ -231,21 +274,21 @@ def _after_piece(name: str, end: int) -> Callable[[int, str], int]:
return step
-def _glob_precedence(pattern: str) -> tuple[tuple[int, int], str]:
- return PatternUtils.calculate_pattern_specificity(pattern), pattern
+def _glob_precedence(pattern: str) -> tuple[int, int, str]:
+ """More literal characters first, then fewer wildcards, so padding a pattern with `*` never makes it win."""
+ wildcards: Final = pattern.count(_GLOB_WILDCARD)
+ return len(pattern) - wildcards, -wildcards, pattern
-def entries_outranked_by(
- entry: str,
- model_max_budget: Collection[str],
- access_group_members: AccessGroupMembers = proxy_access_group_members,
-) -> Iterator[str]:
- """Entries that `entry` can take precedence over for some request model.
+def entries_outranked_by(entry: str, model_max_budget: Collection[str], routed_models: RoutedModels) -> Iterator[str]:
+ """Entries that `entry` can take precedence over for some model a request can name.
- A pattern can match a provider-prefixed or wildcard-routed name whose
- candidates or access groups include any other entry, so patterns are taken
- to overlap: a pattern outranks every less specific pattern, and a model or
- group entry outranks every pattern.
+ Two patterns can both match a name the router has not served yet, so a
+ pattern outranks every less specific pattern. A model or group entry
+ outranks a pattern that matches one of the names it governs: the model,
+ the model under any provider prefix or wildcard route, a served name
+ ending in it, or a member of the group, and every pattern when a member is
+ itself a wildcard route.
"""
if _GLOB_WILDCARD in entry:
entry_precedence: Final = _glob_precedence(entry)
@@ -255,18 +298,41 @@ def entries_outranked_by(
if _GLOB_WILDCARD in other and _glob_precedence(other) < entry_precedence
)
return
- yield from (other for other in model_max_budget if _GLOB_WILDCARD in other)
- entry_members: Final = frozenset(access_group_members(entry))
+ entry_members: Final = routed_models.members(entry)
+ covers_wildcard_route: Final = any(_GLOB_WILDCARD in member for member in entry_members)
+ governed_candidates: Final = frozenset(
+ itertools.chain.from_iterable(
+ _budget_model_candidates(name)
+ for name in _names_governed_by(entry=entry, entry_members=entry_members, routed_models=routed_models)
+ )
+ )
+ yield from (
+ other
+ for other in model_max_budget
+ if _GLOB_WILDCARD in other and (covers_wildcard_route or _glob_matches(other, governed_candidates))
+ )
yield from (
other
for other in model_max_budget
if _GLOB_WILDCARD not in other
and _ranks_ahead_of_group(
- entry=entry, entry_members=entry_members, group=other, group_members=access_group_members(other)
+ entry=entry, entry_members=entry_members, group=other, group_members=routed_models.members(other)
)
)
+def _names_governed_by(entry: str, entry_members: frozenset[str], routed_models: RoutedModels) -> Iterator[str]:
+ yield entry
+ yield from (f"{prefix}{entry}" for prefix in _PROVIDER_PREFIXES)
+ yield from (
+ name
+ for name in routed_models.served_names
+ if _GLOB_WILDCARD not in name and entry in _budget_model_candidates(name)
+ )
+ yield from (name.replace(_GLOB_WILDCARD, entry) for name in routed_models.served_names if _GLOB_WILDCARD in name)
+ yield from (member for member in entry_members if _GLOB_WILDCARD not in member)
+
+
def _ranks_ahead_of_group(entry: str, entry_members: frozenset[str], group: str, group_members: Iterable[str]) -> bool:
return any(
_GLOB_WILDCARD in member
@@ -295,16 +361,21 @@ def _usable_entry(entry: str, raw_budget_config: object) -> ResolvedModelBudget
return ResolvedModelBudget(budget_model=entry, budget_config=budget_config)
-def team_model_budget_applies(
- model: str,
- key_model_max_budget: Mapping[str, object] | None,
- model_access_groups: ModelAccessGroups = proxy_model_access_groups,
-) -> bool:
- """A key entry that spend-gates `model` overrides the team cap: it is then gated on and billed to the key alone."""
+def team_model_budget_applies(model: str, key_model_max_budget: Mapping[str, object] | None) -> bool:
+ """A key entry named after `model` that spend-gates it overrides the team cap: it is then gated on and billed to
+ the key alone. A key pattern or group entry is gated alongside the team cap, never instead of it."""
if not key_model_max_budget:
return True
resolved: Final = resolve_model_budget(
- model=model, model_max_budget=key_model_max_budget, model_access_groups=model_access_groups
+ model=model,
+ model_max_budget=MappingProxyType(
+ {
+ candidate: key_model_max_budget[candidate]
+ for candidate in _budget_model_candidates(model)
+ if candidate in key_model_max_budget
+ }
+ ),
+ model_access_groups=lambda _: frozenset(),
)
return resolved is None or not _spend_gated(resolved.budget_config)
@@ -667,9 +738,7 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
Raises:
BudgetExceededError: If the team has exceeded the model budget
"""
- if not team_model_budget_applies(
- model=model, key_model_max_budget=key_model_max_budget, model_access_groups=self._model_access_groups
- ):
+ if not team_model_budget_applies(model=model, key_model_max_budget=key_model_max_budget):
return True
return await self._is_entity_within_model_budget(
entity_type=Litellm_EntityType.TEAM,
@@ -735,7 +804,7 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
raise litellm.BudgetExceededError(
message=(
exceeded_message
- if resolved.budget_model == model
+ if resolved.budget_model in _budget_model_candidates(model)
else f"{exceeded_message}, budget_model={resolved.budget_model}"
),
current_cost=current_spend,
@@ -784,7 +853,7 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
) -> float:
"""Without Redis each replica's counter holds only its own share, so the window every replica adds to
in the database is the floor."""
- if self.dual_cache.redis_cache is not None:
+ if not self._shares_windows_in_database(entity_type):
return local_spend
shared: Final = await _open_shared_window(
shared_windows=self._shared_windows,
@@ -878,7 +947,6 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
key_model_max_budget=(
key_model_max_budget if isinstance(key_model_max_budget, Mapping) else None
),
- model_access_groups=self._model_access_groups,
)
else None
),
@@ -965,21 +1033,29 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
budget_model=resolved.budget_model,
budget_duration=budget_duration,
)
- await self._join_shared_window(
- spend_key=spend_key, start_time_key=start_time_key, budget_duration=budget_duration
- )
+ shares_window: Final = self._shares_windows_in_database(entity_type)
+ if shares_window:
+ await self._join_shared_window(
+ spend_key=spend_key, start_time_key=start_time_key, budget_duration=budget_duration
+ )
await self._increment_spend_for_key(
budget_config=resolved.budget_config,
spend_key=spend_key,
start_time_key=start_time_key,
response_cost=response_cost,
)
- await self._share_spend(
- spend_key=spend_key,
- start_time_key=start_time_key,
- budget_duration=budget_duration,
- response_cost=response_cost,
- )
+ if shares_window:
+ await self._share_spend(
+ spend_key=spend_key,
+ start_time_key=start_time_key,
+ budget_duration=budget_duration,
+ response_cost=response_cost,
+ )
+
+ def _shares_windows_in_database(self, entity_type: Litellm_EntityType) -> bool:
+ """End-user ids are whatever the client sends, so their windows stay on each replica rather than adding a
+ database row per id."""
+ return self.dual_cache.redis_cache is None and entity_type not in _PER_REPLICA_SCOPES
async def _join_shared_window(self, spend_key: str, start_time_key: str, budget_duration: str) -> None:
"""Without Redis, start this replica's window where the database says every replica's window starts.
@@ -987,8 +1063,6 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
Every replica then sends the same window start with its spend, so no flush reads as a newer window that
replaces the spend the others already added.
"""
- if self.dual_cache.redis_cache is not None:
- return
now: Final = self._clock()
local: Final = self.dual_cache.in_memory_cache
if _open_local_window_start(
@@ -1014,8 +1088,6 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
async def _share_spend(
self, spend_key: str, start_time_key: str, budget_duration: str, response_cost: float
) -> None:
- if self.dual_cache.redis_cache is not None:
- return
now: Final = self._clock()
await self._shared_windows.add(
spend_key=spend_key,
diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py
index c1b81a345d0..89c0d5e7b56 100644
--- a/litellm/proxy/management_endpoints/team_endpoints.py
+++ b/litellm/proxy/management_endpoints/team_endpoints.py
@@ -137,10 +137,10 @@ from litellm.proxy.common_utils.user_api_key_cache import (
from litellm.proxy.db.db_span import db_span
from litellm.proxy.hooks.key_management_event_hooks import KeyManagementEventHooks
from litellm.proxy.hooks.model_max_budget_limiter import (
- AccessGroupMembers,
+ RoutedModels,
build_model_max_budget_usage,
entries_outranked_by,
- proxy_access_group_members,
+ proxy_routed_models,
resolve_model_budget,
)
from litellm.proxy.management.teams.authz import TEAM_OR_ORG_ADMIN, TeamRole, is_team_admin, team_access_denied
@@ -1430,7 +1430,7 @@ def _check_team_model_budget_update_authority(
data: UpdateTeamRequest,
user_api_key_dict: UserAPIKeyAuth,
existing_model_max_budget: Mapping[str, object] | None,
- access_group_members: AccessGroupMembers = proxy_access_group_members,
+ routed_models_source: Callable[[], RoutedModels] = proxy_routed_models,
) -> None:
"""Like `_check_team_budget_update_authority`: only a proxy admin may raise, re-window or drop a per-model cap."""
if user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN:
@@ -1470,12 +1470,13 @@ def _check_team_model_budget_update_authority(
)
},
)
+ routed_models: Final = routed_models_source()
for model_name, proposed in requested.items():
_check_outranked_model_caps(
entry=model_name,
proposed=proposed,
existing_model_max_budget=existing_model_max_budget,
- access_group_members=access_group_members,
+ routed_models=routed_models,
)
@@ -1483,12 +1484,12 @@ def _check_outranked_model_caps(
entry: str,
proposed: BudgetConfig,
existing_model_max_budget: Mapping[str, object],
- access_group_members: AccessGroupMembers,
+ routed_models: RoutedModels,
) -> None:
if _keeps_or_tightens(proposed=proposed, existing=_existing_model_cap(existing_model_max_budget.get(entry))):
return
outranked: Final = next(
- _looser_than_outranked_caps(entry, proposed, existing_model_max_budget, access_group_members), None
+ _looser_than_outranked_caps(entry, proposed, existing_model_max_budget, routed_models), None
)
if outranked is None:
return
@@ -1510,12 +1511,12 @@ def _looser_than_outranked_caps(
entry: str,
proposed: BudgetConfig,
existing_model_max_budget: Mapping[str, object],
- access_group_members: AccessGroupMembers,
+ routed_models: RoutedModels,
) -> Iterator[tuple[str, BudgetConfig]]:
outranked_caps: Final = (
(outranked, _existing_model_cap(existing_model_max_budget[outranked]))
for outranked in entries_outranked_by(
- entry=entry, model_max_budget=existing_model_max_budget, access_group_members=access_group_members
+ entry=entry, model_max_budget=existing_model_max_budget, routed_models=routed_models
)
)
yield from (
diff --git a/tests/unit/proxy/hooks/test_model_budget_shared_windows.py b/tests/unit/proxy/hooks/test_model_budget_shared_windows.py
index 8b103c33d32..c294d667315 100644
--- a/tests/unit/proxy/hooks/test_model_budget_shared_windows.py
+++ b/tests/unit/proxy/hooks/test_model_budget_shared_windows.py
@@ -35,6 +35,16 @@ class _Database:
return 1
+class _OutracedDatabase:
+ def __init__(self, table: _Table, winner: object) -> None:
+ self.litellm_budgetwindowspend = table
+ self._winner = winner
+
+ async def execute_raw(self, query: str, *params: object) -> int:
+ self.litellm_budgetwindowspend.row = self._winner
+ return 0
+
+
class _Sink:
def __init__(self) -> None:
self.transactions: tuple[WindowSpendTransaction, ...] = ()
@@ -143,6 +153,20 @@ async def test_open_reads_the_window_from_the_writer_while_the_read_replica_lags
assert started_at == WINDOW_START.timestamp()
+@pytest.mark.asyncio
+async def test_open_joins_the_window_a_racing_replica_opened_first():
+ winner: Final = SimpleNamespace(window_start=WINDOW_START.replace(tzinfo=None), spend=2.5)
+ database: Final = _OutracedDatabase(_Table(), winner)
+ windows: Final = DatabaseSharedWindows(
+ prisma_client_source=lambda: SimpleNamespace(db=database, writer_db=database), # pyright: ignore[reportArgumentType] # duck-typed prisma double
+ sink=_Sink(),
+ )
+
+ started_at: Final = await windows.open(spend_key=SPEND_KEY, budget_duration="1d", now=WINDOW_START.timestamp() + 60)
+
+ assert started_at == WINDOW_START.timestamp()
+
+
@pytest.mark.asyncio
@pytest.mark.parametrize("stored_start", [None, WINDOW_START - timedelta(days=2)])
async def test_open_starts_a_window_at_now_when_none_is_open(stored_start: datetime | None):
diff --git a/tests/unit/proxy/hooks/test_model_max_budget_limiter.py b/tests/unit/proxy/hooks/test_model_max_budget_limiter.py
index cf925273163..e97be8abbb0 100644
--- a/tests/unit/proxy/hooks/test_model_max_budget_limiter.py
+++ b/tests/unit/proxy/hooks/test_model_max_budget_limiter.py
@@ -16,10 +16,14 @@ from litellm.proxy.hooks.model_budget_shared_windows import SharedWindow, Shared
from litellm.proxy.hooks.model_max_budget_limiter import (
PROXY_VirtualKeyModelMaxBudgetLimiter,
build_model_max_budget_usage,
+ model_access_groups_of,
model_budget_spend_cache_key,
resolve_model_budget,
+ routed_models_of,
team_member_budget_entity_id,
+ team_model_budget_applies,
)
+from litellm.router import Router
from litellm.types.caching import RedisPipelineIncrementOperation
from litellm.types.utils import LiteLLMBatch, Usage
@@ -453,6 +457,7 @@ async def test_a_model_added_to_the_access_group_later_is_covered_with_no_key_ch
({"claude-opus-4*": FAMILY_CAP, "claude-opus*": ROOMY_CAP}, "claude-opus-4-7", True),
({"claude-opus-4*": FAMILY_CAP, "claude-opus*": ROOMY_CAP}, "claude-opus-5-5", False),
({"claude-opus-4*": {"budget_limit": CHAT_COST}, "claude-opus*": FAMILY_CAP}, "claude-opus-5-5", True),
+ ({"claude-opus-4*": FAMILY_CAP, "c" + "*" * 15: ROOMY_CAP}, "claude-opus-4-7", True),
],
)
async def test_the_most_specific_entry_takes_the_spend_and_gates_the_request(
@@ -469,6 +474,87 @@ async def test_the_most_specific_entry_takes_the_spend_and_gates_the_request(
assert await limiter.is_key_within_model_budget(user_api_key_dict=_key(budget), model=checked_model)
+@pytest.mark.asyncio
+async def test_an_exact_entry_refusal_reads_as_before_for_a_provider_prefixed_model():
+ budget: Final = {"claude-opus-4-8": FAMILY_CAP}
+ limiter: Final = _family_limiter()
+
+ await _spend_on(limiter, model="claude-opus-4-8", model_max_budget=budget)
+
+ with pytest.raises(litellm.BudgetExceededError, match=r"model=anthropic/claude-opus-4-8$"):
+ await limiter.is_key_within_model_budget(user_api_key_dict=_key(budget), model="anthropic/claude-opus-4-8")
+
+
+@pytest.mark.parametrize(
+ ("key_entry", "team_cap_applies"),
+ [("claude-opus-4-8", False), ("anthropic/claude-opus-4-8", True), ("claude-opus*", True), (OPUS_GROUP, True)],
+)
+def test_only_a_key_entry_named_after_the_model_replaces_the_team_cap(key_entry: str, team_cap_applies: bool):
+ assert team_model_budget_applies(model="claude-opus-4-8", key_model_max_budget={key_entry: ROOMY_CAP}) is (
+ team_cap_applies
+ )
+
+
+@pytest.mark.asyncio
+async def test_end_user_windows_stay_on_the_replica_without_redis():
+ budget: Final = {"claude-opus*": FAMILY_CAP}
+ table: Final = _SharedWindowsDouble()
+ limiter: Final = _family_limiter(shared_windows=table)
+
+ await _log_success(
+ limiter,
+ {
+ "call_type": "acompletion",
+ "standard_logging_object": {
+ "call_type": "acompletion",
+ "response_cost": CHAT_COST,
+ "model": "anthropic/claude-opus-4-8",
+ "model_group": "claude-opus-4-8",
+ "end_user": "end-user-1",
+ "metadata": {},
+ },
+ "litellm_params": {"metadata": {"user_api_key_end_user_model_max_budget": budget}},
+ },
+ )
+ table.flush()
+
+ assert dict(table.rows) == {}
+ with pytest.raises(litellm.BudgetExceededError, match="model=claude-opus-5-5"):
+ await limiter.is_end_user_within_model_budget(
+ end_user_id="end-user-1", end_user_model_max_budget=budget, model="claude-opus-5-5"
+ )
+
+
+def test_an_access_group_named_after_a_served_model_never_widens_that_models_entry():
+ router: Final = Router(
+ model_list=[
+ {
+ "model_name": "gpt-4o",
+ "litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-test"},
+ "model_info": {"access_groups": ["gpt-4o", "openai-models"]},
+ },
+ {
+ "model_name": "gpt-4o-mini",
+ "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-test"},
+ "model_info": {"access_groups": ["gpt-4o", "openai-models"]},
+ },
+ ]
+ )
+
+ def governing_entry(model_max_budget: Mapping[str, object]) -> str | None:
+ resolved: Final = resolve_model_budget(
+ model="gpt-4o-mini",
+ model_max_budget=model_max_budget,
+ model_access_groups=lambda model: model_access_groups_of(router=router, model=model),
+ )
+ return None if resolved is None else resolved.budget_model
+
+ assert governing_entry({"gpt-4o": FAMILY_CAP}) is None
+ assert governing_entry({"openai-models": FAMILY_CAP}) == "openai-models"
+ assert routed_models_of(router).members("gpt-4o") == frozenset()
+ assert routed_models_of(router).members("openai-models") == frozenset({"gpt-4o", "gpt-4o-mini"})
+
+
@pytest.mark.asyncio
async def test_key_info_usage_reports_the_family_total():
budget: Final = {"claude-opus*": ROOMY_CAP}
diff --git a/tests/unit/proxy/management_endpoints/test_team_endpoints.py b/tests/unit/proxy/management_endpoints/test_team_endpoints.py
index 2d728b506e9..7fbb639648e 100644
--- a/tests/unit/proxy/management_endpoints/test_team_endpoints.py
+++ b/tests/unit/proxy/management_endpoints/test_team_endpoints.py
@@ -44,6 +44,7 @@ from litellm.proxy._types import (
UpdateTeamRequest,
UserAPIKeyAuth, # Import UserAPIKeyAuth
)
+from litellm.proxy.hooks.model_max_budget_limiter import RoutedModels
from litellm.proxy.management.teams.authz import TeamAccess
from litellm.proxy.management_endpoints.team_admin_field_permissions import (
PROXY_CONTROLLED_TEAM_METADATA_KEYS,
@@ -17121,21 +17122,23 @@ def test_team_admin_can_tighten_or_keep_team_model_caps(requested) -> None:
_TEAM_OPUS_FAMILY_CAP: Final = MappingProxyType({"claude-opus*": {"max_budget": 10.0, "budget_duration": "1d"}})
_TEAM_OPUS_GROUP_CAP: Final = MappingProxyType({"z-opus-group": {"max_budget": 10.0, "budget_duration": "1d"}})
-_TEAM_ACCESS_GROUP_MEMBERS: Final = MappingProxyType(
- {
- "z-opus-group": ("claude-opus-4-8",),
- "a-opus-group": ("claude-opus-4-8", "claude-opus-5-5"),
- "zz-opus-group": ("claude-opus-4-8",),
- "a-gemini-group": ("gemini-3-pro",),
- "z-anthropic-group": ("anthropic/*",),
- }
+_TEAM_ROUTED_MODELS: Final = RoutedModels(
+ served_names=frozenset(
+ {"claude-opus-4-8", "claude-opus-5-5", "gemini-3-pro", "gpt-4o", "my-team/gpt-4o", "team-route/*"}
+ ),
+ group_members=MappingProxyType(
+ {
+ "opus-group": frozenset({"claude-opus-4-8"}),
+ "z-opus-group": frozenset({"claude-opus-4-8"}),
+ "a-opus-group": frozenset({"claude-opus-4-8", "claude-opus-5-5"}),
+ "zz-opus-group": frozenset({"claude-opus-4-8"}),
+ "a-gemini-group": frozenset({"gemini-3-pro"}),
+ "z-anthropic-group": frozenset({"anthropic/*"}),
+ }
+ ),
)
-def _team_access_group_members(access_group: str) -> tuple[str, ...]:
- return _TEAM_ACCESS_GROUP_MEMBERS.get(access_group, ())
-
-
def _check_team_admin_model_caps(existing: Mapping[str, object], requested: Mapping[str, object]) -> None:
from litellm.proxy.management_endpoints.team_endpoints import _check_team_model_budget_update_authority
@@ -17143,7 +17146,7 @@ def _check_team_admin_model_caps(existing: Mapping[str, object], requested: Mapp
data=UpdateTeamRequest(team_id="t1", model_max_budget=dict(requested)),
user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, user_id="team-admin"),
existing_model_max_budget=existing,
- access_group_members=_team_access_group_members,
+ routed_models_source=lambda: _TEAM_ROUTED_MODELS,
)
@@ -17159,6 +17162,19 @@ def _check_team_admin_model_caps(existing: Mapping[str, object], requested: Mapp
{"z-anthropic-group": {"max_budget": 10.0, "budget_duration": "1d"}},
{"claude-opus-4-8": {"max_budget": 1000.0, "budget_duration": "1d"}},
),
+ (_TEAM_OPUS_FAMILY_CAP, {"claude-opus-4-8": {"max_budget": 1000.0, "budget_duration": "1d"}}),
+ (
+ {"anthropic/*": {"max_budget": 10.0, "budget_duration": "1d"}},
+ {"claude-opus-4-8": {"max_budget": 1000.0, "budget_duration": "1d"}},
+ ),
+ (
+ {"my-team/*": {"max_budget": 10.0, "budget_duration": "1d"}},
+ {"gpt-4o": {"max_budget": 1000.0, "budget_duration": "1d"}},
+ ),
+ (
+ {"team-route/*": {"max_budget": 10.0, "budget_duration": "1d"}},
+ {"gpt-4o": {"max_budget": 1000.0, "budget_duration": "1d"}},
+ ),
],
ids=[
"group_over_pattern",
@@ -17167,6 +17183,10 @@ def _check_team_admin_model_caps(existing: Mapping[str, object], requested: Mapp
"earlier_group_sharing_a_model",
"model_inside_group",
"model_under_wildcard_group",
+ "model_matching_pattern",
+ "model_under_provider_pattern",
+ "model_under_served_prefix",
+ "model_under_wildcard_route",
],
)
def test_team_admin_cannot_add_an_entry_looser_than_a_cap_it_outranks(
@@ -17187,6 +17207,11 @@ def test_team_admin_cannot_add_an_entry_looser_than_a_cap_it_outranks(
(_TEAM_OPUS_GROUP_CAP, {**_TEAM_OPUS_GROUP_CAP, "a-gemini-group": {"max_budget": 1000.0, "budget_duration": "1d"}}),
(_TEAM_OPUS_GROUP_CAP, {**_TEAM_OPUS_GROUP_CAP, "zz-opus-group": {"max_budget": 1000.0, "budget_duration": "1d"}}),
(_TEAM_OPUS_GROUP_CAP, {**_TEAM_OPUS_GROUP_CAP, "gpt-4o": {"max_budget": 1000.0, "budget_duration": "1d"}}),
+ (_TEAM_OPUS_FAMILY_CAP, {**_TEAM_OPUS_FAMILY_CAP, "gpt-4o": {"max_budget": 1000.0, "budget_duration": "1d"}}),
+ (
+ _TEAM_OPUS_FAMILY_CAP,
+ {**_TEAM_OPUS_FAMILY_CAP, "a-gemini-group": {"max_budget": 1000.0, "budget_duration": "1d"}},
+ ),
(
{**_TEAM_OPUS_FAMILY_CAP, "opus-group": {"max_budget": 100.0, "budget_duration": "1d"}},
{**_TEAM_OPUS_FAMILY_CAP, "opus-group": {"max_budget": 100.0, "budget_duration": "1d"}},
@@ -17206,6 +17231,8 @@ def test_team_admin_cannot_add_an_entry_looser_than_a_cap_it_outranks(
"group_without_shared_models",
"later_group_sharing_a_model",
"model_outside_group",
+ "model_outside_pattern",
+ "group_outside_pattern",
"unchanged_looser_entry",
"tightened_looser_entry",
"uncapped_entry_edited",
diff --git a/ui/litellm-dashboard/src/components/key_team_helpers/ModelMaxBudgetEditor.integration.test.tsx b/ui/litellm-dashboard/src/components/key_team_helpers/ModelMaxBudgetEditor.integration.test.tsx
index f7f0acd3254..955b911ca97 100644
--- a/ui/litellm-dashboard/src/components/key_team_helpers/ModelMaxBudgetEditor.integration.test.tsx
+++ b/ui/litellm-dashboard/src/components/key_team_helpers/ModelMaxBudgetEditor.integration.test.tsx
@@ -111,4 +111,22 @@ describe("ModelMaxBudgetEditor model families", () => {
await user.type(newRowModelField, "claude-sonnet*");
expect(await screen.findByRole("option", { name: /claude-sonnet\*/ })).toBeInTheDocument();
});
+
+ it("does not offer a typed name that is neither listed nor a pattern", async () => {
+ const user = userEvent.setup();
+ renderWithProviders(
+ ,
+ );
+
+ await user.click(addButton());
+ await user.type(screen.getByPlaceholderText("Select model"), "claude-opsu-4-8");
+
+ expect(screen.queryByRole("option", { name: /claude-opsu-4-8/ })).not.toBeInTheDocument();
+ expect(screen.getByText("No models found")).toBeInTheDocument();
+ });
});
diff --git a/ui/litellm-dashboard/src/components/key_team_helpers/ModelMaxBudgetEditor.tsx b/ui/litellm-dashboard/src/components/key_team_helpers/ModelMaxBudgetEditor.tsx
index 304d0a6ad1d..1e321a59e02 100644
--- a/ui/litellm-dashboard/src/components/key_team_helpers/ModelMaxBudgetEditor.tsx
+++ b/ui/litellm-dashboard/src/components/key_team_helpers/ModelMaxBudgetEditor.tsx
@@ -50,6 +50,8 @@ export const MODEL_BUDGET_PERIOD_OPTIONS = [
const DEFAULT_PERIOD = "30d";
+const MODEL_PATTERN_VALUE = { hint: "Model pattern", accepts: (typed: string) => typed.includes("*") };
+
export const entriesToModelMaxBudget = (entries: readonly ModelBudgetEntry[]): ModelMaxBudget =>
Object.fromEntries(
entries
@@ -165,7 +167,7 @@ export function ModelMaxBudgetEditor({
onValueChange={(model) => updateEntry(entry.id, { model })}
placeholder="Select model"
emptyText="No models found"
- customValueHint="Access group or model pattern"
+ customValue={MODEL_PATTERN_VALUE}
reservedValues={entries.flatMap((other) => (other.id !== entry.id && other.model ? [other.model] : []))}
disabled={!premiumUser}
/>
diff --git a/ui/litellm-dashboard/src/components/shared/SearchSelect.integration.test.tsx b/ui/litellm-dashboard/src/components/shared/SearchSelect.integration.test.tsx
index ce5b734941a..ecb8dd161f3 100644
--- a/ui/litellm-dashboard/src/components/shared/SearchSelect.integration.test.tsx
+++ b/ui/litellm-dashboard/src/components/shared/SearchSelect.integration.test.tsx
@@ -63,16 +63,19 @@ describe("SearchSelect", () => {
expect(onValueChange).toHaveBeenLastCalledWith("team-3");
});
- it("offers the typed text as a value only when the caller accepts custom values", async () => {
+ it("offers the typed text as a value only when the caller accepts it", async () => {
const user = userEvent.setup();
const onValueChange = vi.fn();
+ const customValue = { hint: "Custom value", accepts: (typed: string) => typed.endsWith("*") };
const { rerender } = render();
const input = screen.getByRole("combobox");
await user.click(input);
fireEvent.change(input, { target: { value: "team-*" } });
expect(screen.queryByRole("option", { name: /team-\*/ })).not.toBeInTheDocument();
- rerender();
+ rerender();
+ fireEvent.change(input, { target: { value: "team-4" } });
+ expect(screen.queryByRole("option", { name: /team-4/ })).not.toBeInTheDocument();
fireEvent.change(input, { target: { value: "team-*" } });
await user.click(await screen.findByRole("option", { name: /team-\*/ }));
expect(onValueChange).toHaveBeenLastCalledWith("team-*");
diff --git a/ui/litellm-dashboard/src/components/shared/SearchSelect.tsx b/ui/litellm-dashboard/src/components/shared/SearchSelect.tsx
index 9b1203771d3..e139720a2ed 100644
--- a/ui/litellm-dashboard/src/components/shared/SearchSelect.tsx
+++ b/ui/litellm-dashboard/src/components/shared/SearchSelect.tsx
@@ -19,6 +19,11 @@ export interface SearchSelectOption {
icon?: React.ReactNode;
}
+interface CustomValue {
+ hint: string;
+ accepts: (typed: string) => boolean;
+}
+
interface SearchSelectProps {
options: SearchSelectOption[];
value?: string | null;
@@ -29,7 +34,7 @@ interface SearchSelectProps {
className?: string;
inputId?: string;
allowClear?: boolean;
- customValueHint?: string;
+ customValue?: CustomValue;
reservedValues?: readonly string[];
"aria-label"?: string;
id?: string;
@@ -51,7 +56,7 @@ export function SearchSelect({
className,
inputId,
allowClear = true,
- customValueHint,
+ customValue,
reservedValues = [],
"aria-label": ariaLabel,
id,
@@ -64,8 +69,8 @@ export function SearchSelect({
const typed = query.trim();
const isNewValue = typed !== "" && !reservedValues.includes(typed) && !known.some((option) => option.value === typed);
const items =
- customValueHint !== undefined && isNewValue
- ? [...known, { label: typed, value: typed, sublabel: customValueHint }]
+ customValue !== undefined && isNewValue && customValue.accepts(typed)
+ ? [...known, { label: typed, value: typed, sublabel: customValue.hint }]
: known;
return (