mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-22 00:31:44 +00:00
feat(proxy): add model group budgets for virtual keys
Adds a models list to BudgetConfig so a model_max_budget entry can define one shared budget across a group of models. All models in the group draw from a single spend pool keyed by the entry name, enforced and tracked in _PROXY_VirtualKeyModelMaxBudgetLimiter alongside existing per-model budgets. Closes #34367
This commit is contained in:
parent
f7842cdeb7
commit
c27cd0fce2
6 changed files with 238 additions and 15 deletions
|
|
@ -1670,7 +1670,7 @@ class BudgetNewRequest(LiteLLMPydanticObjectBase):
|
|||
)
|
||||
model_max_budget: Optional[GenericBudgetConfigType] = Field(
|
||||
default=None,
|
||||
description="Max budget for each model (e.g. {'gpt-4o': {'max_budget': '0.0000001', 'budget_duration': '1d', 'tpm_limit': 1000, 'rpm_limit': 1000}})",
|
||||
description="Max budget for each model (e.g. {'gpt-4o': {'max_budget': '0.0000001', 'budget_duration': '1d', 'tpm_limit': 1000, 'rpm_limit': 1000}}). An entry with 'models' set shares one budget across that group of models (e.g. {'opus-family': {'models': ['claude-opus-4-5', 'claude-opus-4-6'], 'max_budget': 10, 'budget_duration': '30d'}})",
|
||||
)
|
||||
budget_reset_at: Optional[datetime] = Field(
|
||||
default=None,
|
||||
|
|
|
|||
|
|
@ -58,10 +58,13 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
|
|||
)
|
||||
if _current_model_budget_info is None:
|
||||
verbose_proxy_logger.debug(f"Model {model} not found in internal_model_max_budget")
|
||||
return True
|
||||
|
||||
# check if current model is within budget
|
||||
if _current_model_budget_info.max_budget and _current_model_budget_info.max_budget > 0:
|
||||
if (
|
||||
_current_model_budget_info is not None
|
||||
and _current_model_budget_info.max_budget
|
||||
and _current_model_budget_info.max_budget > 0
|
||||
):
|
||||
_current_spend = await self._get_virtual_key_spend_for_model(
|
||||
user_api_key_hash=user_api_key_dict.token,
|
||||
model=model,
|
||||
|
|
@ -80,6 +83,25 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
|
|||
entity_id=user_api_key_dict.token,
|
||||
)
|
||||
|
||||
for _group_name, _group_budget_info in self._get_matching_model_group_budget_configs(
|
||||
model=model, internal_model_max_budget=internal_model_max_budget
|
||||
):
|
||||
if not _group_budget_info.max_budget or _group_budget_info.max_budget <= 0:
|
||||
continue
|
||||
_group_spend = await self._get_virtual_key_spend_for_model_group(
|
||||
user_api_key_hash=user_api_key_dict.token,
|
||||
model_group_name=_group_name,
|
||||
key_budget_config=_group_budget_info,
|
||||
)
|
||||
if _group_spend is not None and _group_spend > _group_budget_info.max_budget:
|
||||
raise litellm.BudgetExceededError(
|
||||
message=f"LiteLLM Virtual Key: {user_api_key_dict.token}, key_alias: {user_api_key_dict.key_alias}, exceeded budget for model group={_group_name}, model={model}",
|
||||
current_cost=_group_spend,
|
||||
max_budget=_group_budget_info.max_budget,
|
||||
entity_type=Litellm_EntityType.KEY.value,
|
||||
entity_id=user_api_key_dict.token,
|
||||
)
|
||||
|
||||
return True
|
||||
|
||||
async def get_fallback_model_within_budget(
|
||||
|
|
@ -201,6 +223,18 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
|
|||
)
|
||||
return _current_spend
|
||||
|
||||
async def _get_virtual_key_spend_for_model_group(
|
||||
self,
|
||||
user_api_key_hash: str | None,
|
||||
model_group_name: str,
|
||||
key_budget_config: BudgetConfig,
|
||||
) -> float | None:
|
||||
model_group_spend_cache_key = (
|
||||
f"{VIRTUAL_KEY_SPEND_CACHE_KEY_PREFIX}:{user_api_key_hash}:{model_group_name}:"
|
||||
f"{key_budget_config.budget_duration}"
|
||||
)
|
||||
return await self.dual_cache.async_get_cache(key=model_group_spend_cache_key)
|
||||
|
||||
def _get_request_model_budget_config(
|
||||
self, model: str, internal_model_max_budget: GenericBudgetConfigType
|
||||
) -> Optional[BudgetConfig]:
|
||||
|
|
@ -210,10 +244,23 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
|
|||
1. Check if `model` is in `internal_model_max_budget`
|
||||
2. If not, check if `model` without custom llm provider is in `internal_model_max_budget`
|
||||
"""
|
||||
return internal_model_max_budget.get(model, None) or internal_model_max_budget.get(
|
||||
direct_model_max_budget = {
|
||||
_model: _config for _model, _config in internal_model_max_budget.items() if not _config.models
|
||||
}
|
||||
return direct_model_max_budget.get(model, None) or direct_model_max_budget.get(
|
||||
self._get_model_without_custom_llm_provider(model), None
|
||||
)
|
||||
|
||||
def _get_matching_model_group_budget_configs(
|
||||
self, model: str, internal_model_max_budget: GenericBudgetConfigType
|
||||
) -> tuple[tuple[str, BudgetConfig], ...]:
|
||||
model_without_provider = self._get_model_without_custom_llm_provider(model)
|
||||
return tuple(
|
||||
(_group_name, _config)
|
||||
for _group_name, _config in internal_model_max_budget.items()
|
||||
if _config.models and (model in _config.models or model_without_provider in _config.models)
|
||||
)
|
||||
|
||||
def _get_model_without_custom_llm_provider(self, model: str) -> str:
|
||||
if "/" in model:
|
||||
return model.split("/")[-1]
|
||||
|
|
@ -296,6 +343,22 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
|
|||
start_time_key=virtual_start_time_key,
|
||||
response_cost=response_cost,
|
||||
)
|
||||
for _group_name, _group_budget_config in self._get_matching_model_group_budget_configs(
|
||||
model=model, internal_model_max_budget=internal_model_max_budget
|
||||
):
|
||||
if _group_budget_config.budget_duration is None:
|
||||
continue
|
||||
group_spend_key = (
|
||||
f"{VIRTUAL_KEY_SPEND_CACHE_KEY_PREFIX}:{virtual_key}:{_group_name}:"
|
||||
f"{_group_budget_config.budget_duration}"
|
||||
)
|
||||
group_start_time_key = f"virtual_key_budget_start_time:{virtual_key}:{_group_name}"
|
||||
await self._increment_spend_for_key(
|
||||
budget_config=_group_budget_config,
|
||||
spend_key=group_spend_key,
|
||||
start_time_key=group_start_time_key,
|
||||
response_cost=response_cost,
|
||||
)
|
||||
|
||||
if (
|
||||
end_user_id is not None
|
||||
|
|
|
|||
|
|
@ -1507,7 +1507,7 @@ async def generate_key_fn(
|
|||
- disable_global_guardrails: Optional[bool] - Whether to disable global guardrails for the key.
|
||||
- throttle_on_budget_exceeded: Optional[bool] - When the key exceeds its max_budget, throttle its tpm/rpm to the global budget_exceeded_throttle_percentage instead of blocking the key entirely.
|
||||
- permissions: Optional[dict] - key-specific permissions. Currently just used for turning off pii masking (if connected). Example - {"pii": false}
|
||||
- model_max_budget: Optional[Dict[str, BudgetConfig]] - Model-specific budgets {"gpt-4": {"budget_limit": 0.0005, "time_period": "30d"}}}. IF null or {} then no model specific budget.
|
||||
- model_max_budget: Optional[Dict[str, BudgetConfig]] - Model-specific budgets {"gpt-4": {"budget_limit": 0.0005, "time_period": "30d"}}. Set "models" to share one budget across a group of models: {"opus-family": {"models": ["claude-opus-4-5", "claude-opus-4-6"], "budget_limit": 10, "time_period": "30d"}}. IF null or {} then no model specific budget.
|
||||
- budget_fallbacks: Optional[Dict[str, List[str]]] - Per-model fallback chain tried in order when that model's own `model_max_budget` is exceeded, e.g. {"gpt-4o": ["gpt-4o-mini"]}.
|
||||
- model_rpm_limit: Optional[dict] - key-specific model rpm limit. Example - {"text-davinci-002": 1000, "gpt-3.5-turbo": 1000}. IF null or {} then no model specific rpm limit.
|
||||
- model_tpm_limit: Optional[dict] - key-specific model tpm limit. Example - {"text-davinci-002": 1000, "gpt-3.5-turbo": 1000}. IF null or {} then no model specific tpm limit.
|
||||
|
|
@ -1715,7 +1715,7 @@ async def generate_service_account_key_fn(
|
|||
- metadata: Optional[dict] - Metadata for key, store information for key. Example metadata = {"team": "core-infra", "app": "app2", "email": "ishaan@berri.ai" }
|
||||
- guardrails: Optional[List[str]] - List of active guardrails for the key
|
||||
- permissions: Optional[dict] - key-specific permissions. Currently just used for turning off pii masking (if connected). Example - {"pii": false}
|
||||
- model_max_budget: Optional[Dict[str, BudgetConfig]] - Model-specific budgets {"gpt-4": {"budget_limit": 0.0005, "time_period": "30d"}}}. IF null or {} then no model specific budget.
|
||||
- model_max_budget: Optional[Dict[str, BudgetConfig]] - Model-specific budgets {"gpt-4": {"budget_limit": 0.0005, "time_period": "30d"}}. Set "models" to share one budget across a group of models: {"opus-family": {"models": ["claude-opus-4-5", "claude-opus-4-6"], "budget_limit": 10, "time_period": "30d"}}. IF null or {} then no model specific budget.
|
||||
- budget_fallbacks: Optional[Dict[str, List[str]]] - Per-model fallback chain tried in order when that model's own `model_max_budget` is exceeded, e.g. {"gpt-4o": ["gpt-4o-mini"]}.
|
||||
- model_rpm_limit: Optional[dict] - key-specific model rpm limit. Example - {"text-davinci-002": 1000, "gpt-3.5-turbo": 1000}. IF null or {} then no model specific rpm limit.
|
||||
- model_tpm_limit: Optional[dict] - key-specific model tpm limit. Example - {"text-davinci-002": 1000, "gpt-3.5-turbo": 1000}. IF null or {} then no model specific tpm limit.
|
||||
|
|
@ -2521,7 +2521,7 @@ async def update_key_fn(
|
|||
- enforced_params: Optional[List[str]] - List of enforced params for the key (Enterprise only). [Docs](https://docs.litellm.ai/docs/proxy/enterprise#enforce-required-params-for-llm-requests)
|
||||
- spend: Optional[float] - Amount spent by key
|
||||
- max_budget: Optional[float] - Max budget for key
|
||||
- model_max_budget: Optional[Dict[str, BudgetConfig]] - Model-specific budgets {"gpt-4": {"budget_limit": 0.0005, "time_period": "30d"}}
|
||||
- model_max_budget: Optional[Dict[str, BudgetConfig]] - Model-specific budgets {"gpt-4": {"budget_limit": 0.0005, "time_period": "30d"}}. Set "models" to share one budget across a group of models: {"opus-family": {"models": ["claude-opus-4-5", "claude-opus-4-6"], "budget_limit": 10, "time_period": "30d"}}
|
||||
- budget_fallbacks: Optional[Dict[str, List[str]]] - Per-model fallback chain tried in order when that model's own `model_max_budget` is exceeded, e.g. {"gpt-4o": ["gpt-4o-mini"]}.
|
||||
- budget_duration: Optional[str] - Budget reset period ("30d", "1h", etc.)
|
||||
- soft_budget: Optional[float] - [TODO] Soft budget limit (warning vs. hard stop). Will trigger a slack alert when this soft budget is reached.
|
||||
|
|
@ -4584,7 +4584,7 @@ async def regenerate_key_fn(
|
|||
- tags: Optional[List[str]] - Tags for organizing keys (Enterprise only)
|
||||
- spend: Optional[float] - Amount spent by key
|
||||
- max_budget: Optional[float] - Max budget for key
|
||||
- model_max_budget: Optional[Dict[str, BudgetConfig]] - Model-specific budgets {"gpt-4": {"budget_limit": 0.0005, "time_period": "30d"}}
|
||||
- model_max_budget: Optional[Dict[str, BudgetConfig]] - Model-specific budgets {"gpt-4": {"budget_limit": 0.0005, "time_period": "30d"}}. Set "models" to share one budget across a group of models: {"opus-family": {"models": ["claude-opus-4-5", "claude-opus-4-6"], "budget_limit": 10, "time_period": "30d"}}
|
||||
- budget_fallbacks: Optional[Dict[str, List[str]]] - Per-model fallback chain tried in order when that model's own `model_max_budget` is exceeded, e.g. {"gpt-4o": ["gpt-4o-mini"]}.
|
||||
- budget_duration: Optional[str] - Budget reset period ("30d", "1h", etc.)
|
||||
- soft_budget: Optional[float] - Soft budget limit (warning vs. hard stop). Will trigger a slack alert when this soft budget is reached.
|
||||
|
|
|
|||
|
|
@ -3333,6 +3333,7 @@ class BudgetConfig(BaseModel):
|
|||
budget_duration: Optional[str] = None
|
||||
tpm_limit: Optional[int] = None
|
||||
rpm_limit: Optional[int] = None
|
||||
models: Optional[List[str]] = None
|
||||
|
||||
def __init__(self, **data: Any) -> None:
|
||||
# Map time_period to budget_duration if present
|
||||
|
|
|
|||
157
tests/test_litellm/proxy/hooks/test_model_max_budget_limiter.py
Normal file
157
tests/test_litellm/proxy/hooks/test_model_max_budget_limiter.py
Normal file
|
|
@ -0,0 +1,157 @@
|
|||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm.caching.caching import DualCache
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.hooks.model_max_budget_limiter import (
|
||||
_PROXY_VirtualKeyModelMaxBudgetLimiter,
|
||||
)
|
||||
|
||||
VIRTUAL_KEY = "test-key-hash"
|
||||
|
||||
OPUS_GROUP_BUDGET = {
|
||||
"opus-family": {
|
||||
"models": ["anthropic-opus-4-7", "anthropic-opus-4-8"],
|
||||
"budget_limit": 10.0,
|
||||
"time_period": "30d",
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def _make_limiter() -> _PROXY_VirtualKeyModelMaxBudgetLimiter:
|
||||
return _PROXY_VirtualKeyModelMaxBudgetLimiter(dual_cache=DualCache())
|
||||
|
||||
|
||||
def _make_key(model_max_budget: dict) -> UserAPIKeyAuth:
|
||||
return UserAPIKeyAuth(
|
||||
token=VIRTUAL_KEY,
|
||||
key_alias="test-alias",
|
||||
model_max_budget=model_max_budget,
|
||||
)
|
||||
|
||||
|
||||
async def _log_spend(
|
||||
limiter: _PROXY_VirtualKeyModelMaxBudgetLimiter,
|
||||
model: str,
|
||||
response_cost: float,
|
||||
model_max_budget: dict,
|
||||
) -> None:
|
||||
kwargs = {
|
||||
"standard_logging_object": {
|
||||
"response_cost": response_cost,
|
||||
"model": model,
|
||||
"metadata": {"user_api_key_hash": VIRTUAL_KEY},
|
||||
},
|
||||
"litellm_params": {"metadata": {"user_api_key_model_max_budget": model_max_budget}},
|
||||
}
|
||||
await limiter.async_log_success_event(kwargs, response_obj=None, start_time=None, end_time=None)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_group_budget_shared_across_models():
|
||||
limiter = _make_limiter()
|
||||
key = _make_key(OPUS_GROUP_BUDGET)
|
||||
|
||||
await _log_spend(limiter, "anthropic-opus-4-7", 11.0, OPUS_GROUP_BUDGET)
|
||||
|
||||
with pytest.raises(litellm.BudgetExceededError, match="model group=opus-family"):
|
||||
await limiter.is_key_within_model_budget(key, "anthropic-opus-4-7")
|
||||
with pytest.raises(litellm.BudgetExceededError, match="model group=opus-family"):
|
||||
await limiter.is_key_within_model_budget(key, "anthropic-opus-4-8")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_group_budget_ignores_models_outside_group():
|
||||
limiter = _make_limiter()
|
||||
key = _make_key(OPUS_GROUP_BUDGET)
|
||||
|
||||
await _log_spend(limiter, "anthropic-opus-4-7", 11.0, OPUS_GROUP_BUDGET)
|
||||
|
||||
assert await limiter.is_key_within_model_budget(key, "anthropic-sonnet-5") is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_group_budget_within_budget_passes():
|
||||
limiter = _make_limiter()
|
||||
key = _make_key(OPUS_GROUP_BUDGET)
|
||||
|
||||
await _log_spend(limiter, "anthropic-opus-4-7", 4.0, OPUS_GROUP_BUDGET)
|
||||
await _log_spend(limiter, "anthropic-opus-4-8", 6.0, OPUS_GROUP_BUDGET)
|
||||
|
||||
assert await limiter.is_key_within_model_budget(key, "anthropic-opus-4-7") is True
|
||||
assert await limiter.is_key_within_model_budget(key, "anthropic-opus-4-8") is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_group_spend_accumulates_across_models_in_one_pool():
|
||||
limiter = _make_limiter()
|
||||
key = _make_key(OPUS_GROUP_BUDGET)
|
||||
|
||||
await _log_spend(limiter, "anthropic-opus-4-7", 6.0, OPUS_GROUP_BUDGET)
|
||||
assert await limiter.is_key_within_model_budget(key, "anthropic-opus-4-8") is True
|
||||
|
||||
await _log_spend(limiter, "anthropic-opus-4-8", 6.0, OPUS_GROUP_BUDGET)
|
||||
with pytest.raises(litellm.BudgetExceededError, match="model group=opus-family"):
|
||||
await limiter.is_key_within_model_budget(key, "anthropic-opus-4-7")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_group_budget_matches_provider_prefixed_model():
|
||||
limiter = _make_limiter()
|
||||
key = _make_key(OPUS_GROUP_BUDGET)
|
||||
|
||||
await _log_spend(limiter, "anthropic-opus-4-7", 11.0, OPUS_GROUP_BUDGET)
|
||||
|
||||
with pytest.raises(litellm.BudgetExceededError, match="model group=opus-family"):
|
||||
await limiter.is_key_within_model_budget(key, "anthropic/anthropic-opus-4-8")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_group_entry_is_not_a_direct_model_budget():
|
||||
limiter = _make_limiter()
|
||||
key = _make_key(OPUS_GROUP_BUDGET)
|
||||
|
||||
await _log_spend(limiter, "anthropic-opus-4-7", 11.0, OPUS_GROUP_BUDGET)
|
||||
|
||||
assert await limiter.is_key_within_model_budget(key, "opus-family") is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_direct_and_group_budgets_are_both_enforced():
|
||||
model_max_budget = {
|
||||
"anthropic-opus-4-7": {"budget_limit": 1.0, "time_period": "30d"},
|
||||
**OPUS_GROUP_BUDGET,
|
||||
}
|
||||
limiter = _make_limiter()
|
||||
key = _make_key(model_max_budget)
|
||||
|
||||
await _log_spend(limiter, "anthropic-opus-4-7", 2.0, model_max_budget)
|
||||
|
||||
with pytest.raises(litellm.BudgetExceededError, match="model=anthropic-opus-4-7"):
|
||||
await limiter.is_key_within_model_budget(key, "anthropic-opus-4-7")
|
||||
assert await limiter.is_key_within_model_budget(key, "anthropic-opus-4-8") is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_group_budget_exceeded_even_when_direct_budget_is_fine():
|
||||
model_max_budget = {
|
||||
"anthropic-opus-4-7": {"budget_limit": 100.0, "time_period": "30d"},
|
||||
**OPUS_GROUP_BUDGET,
|
||||
}
|
||||
limiter = _make_limiter()
|
||||
key = _make_key(model_max_budget)
|
||||
|
||||
await _log_spend(limiter, "anthropic-opus-4-8", 11.0, model_max_budget)
|
||||
|
||||
with pytest.raises(litellm.BudgetExceededError, match="model group=opus-family"):
|
||||
await limiter.is_key_within_model_budget(key, "anthropic-opus-4-7")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_spend_at_exactly_group_budget_passes():
|
||||
limiter = _make_limiter()
|
||||
key = _make_key(OPUS_GROUP_BUDGET)
|
||||
|
||||
await _log_spend(limiter, "anthropic-opus-4-7", 10.0, OPUS_GROUP_BUDGET)
|
||||
|
||||
assert await limiter.is_key_within_model_budget(key, "anthropic-opus-4-8") is True
|
||||
16
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
16
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -6565,7 +6565,7 @@ export interface paths {
|
|||
* - disable_global_guardrails: Optional[bool] - Whether to disable global guardrails for the key.
|
||||
* - throttle_on_budget_exceeded: Optional[bool] - When the key exceeds its max_budget, throttle its tpm/rpm to the global budget_exceeded_throttle_percentage instead of blocking the key entirely.
|
||||
* - permissions: Optional[dict] - key-specific permissions. Currently just used for turning off pii masking (if connected). Example - {"pii": false}
|
||||
* - model_max_budget: Optional[Dict[str, BudgetConfig]] - Model-specific budgets {"gpt-4": {"budget_limit": 0.0005, "time_period": "30d"}}}. IF null or {} then no model specific budget.
|
||||
* - model_max_budget: Optional[Dict[str, BudgetConfig]] - Model-specific budgets {"gpt-4": {"budget_limit": 0.0005, "time_period": "30d"}}. Set "models" to share one budget across a group of models: {"opus-family": {"models": ["claude-opus-4-5", "claude-opus-4-6"], "budget_limit": 10, "time_period": "30d"}}. IF null or {} then no model specific budget.
|
||||
* - budget_fallbacks: Optional[Dict[str, List[str]]] - Per-model fallback chain tried in order when that model's own `model_max_budget` is exceeded, e.g. {"gpt-4o": ["gpt-4o-mini"]}.
|
||||
* - model_rpm_limit: Optional[dict] - key-specific model rpm limit. Example - {"text-davinci-002": 1000, "gpt-3.5-turbo": 1000}. IF null or {} then no model specific rpm limit.
|
||||
* - model_tpm_limit: Optional[dict] - key-specific model tpm limit. Example - {"text-davinci-002": 1000, "gpt-3.5-turbo": 1000}. IF null or {} then no model specific tpm limit.
|
||||
|
|
@ -6773,7 +6773,7 @@ export interface paths {
|
|||
* - tags: Optional[List[str]] - Tags for organizing keys (Enterprise only)
|
||||
* - spend: Optional[float] - Amount spent by key
|
||||
* - max_budget: Optional[float] - Max budget for key
|
||||
* - model_max_budget: Optional[Dict[str, BudgetConfig]] - Model-specific budgets {"gpt-4": {"budget_limit": 0.0005, "time_period": "30d"}}
|
||||
* - model_max_budget: Optional[Dict[str, BudgetConfig]] - Model-specific budgets {"gpt-4": {"budget_limit": 0.0005, "time_period": "30d"}}. Set "models" to share one budget across a group of models: {"opus-family": {"models": ["claude-opus-4-5", "claude-opus-4-6"], "budget_limit": 10, "time_period": "30d"}}
|
||||
* - budget_fallbacks: Optional[Dict[str, List[str]]] - Per-model fallback chain tried in order when that model's own `model_max_budget` is exceeded, e.g. {"gpt-4o": ["gpt-4o-mini"]}.
|
||||
* - budget_duration: Optional[str] - Budget reset period ("30d", "1h", etc.)
|
||||
* - soft_budget: Optional[float] - Soft budget limit (warning vs. hard stop). Will trigger a slack alert when this soft budget is reached.
|
||||
|
|
@ -6849,7 +6849,7 @@ export interface paths {
|
|||
* - metadata: Optional[dict] - Metadata for key, store information for key. Example metadata = {"team": "core-infra", "app": "app2", "email": "ishaan@berri.ai" }
|
||||
* - guardrails: Optional[List[str]] - List of active guardrails for the key
|
||||
* - permissions: Optional[dict] - key-specific permissions. Currently just used for turning off pii masking (if connected). Example - {"pii": false}
|
||||
* - model_max_budget: Optional[Dict[str, BudgetConfig]] - Model-specific budgets {"gpt-4": {"budget_limit": 0.0005, "time_period": "30d"}}}. IF null or {} then no model specific budget.
|
||||
* - model_max_budget: Optional[Dict[str, BudgetConfig]] - Model-specific budgets {"gpt-4": {"budget_limit": 0.0005, "time_period": "30d"}}. Set "models" to share one budget across a group of models: {"opus-family": {"models": ["claude-opus-4-5", "claude-opus-4-6"], "budget_limit": 10, "time_period": "30d"}}. IF null or {} then no model specific budget.
|
||||
* - budget_fallbacks: Optional[Dict[str, List[str]]] - Per-model fallback chain tried in order when that model's own `model_max_budget` is exceeded, e.g. {"gpt-4o": ["gpt-4o-mini"]}.
|
||||
* - model_rpm_limit: Optional[dict] - key-specific model rpm limit. Example - {"text-davinci-002": 1000, "gpt-3.5-turbo": 1000}. IF null or {} then no model specific rpm limit.
|
||||
* - model_tpm_limit: Optional[dict] - key-specific model tpm limit. Example - {"text-davinci-002": 1000, "gpt-3.5-turbo": 1000}. IF null or {} then no model specific tpm limit.
|
||||
|
|
@ -6948,7 +6948,7 @@ export interface paths {
|
|||
* - enforced_params: Optional[List[str]] - List of enforced params for the key (Enterprise only). [Docs](https://docs.litellm.ai/docs/proxy/enterprise#enforce-required-params-for-llm-requests)
|
||||
* - spend: Optional[float] - Amount spent by key
|
||||
* - max_budget: Optional[float] - Max budget for key
|
||||
* - model_max_budget: Optional[Dict[str, BudgetConfig]] - Model-specific budgets {"gpt-4": {"budget_limit": 0.0005, "time_period": "30d"}}
|
||||
* - model_max_budget: Optional[Dict[str, BudgetConfig]] - Model-specific budgets {"gpt-4": {"budget_limit": 0.0005, "time_period": "30d"}}. Set "models" to share one budget across a group of models: {"opus-family": {"models": ["claude-opus-4-5", "claude-opus-4-6"], "budget_limit": 10, "time_period": "30d"}}
|
||||
* - budget_fallbacks: Optional[Dict[str, List[str]]] - Per-model fallback chain tried in order when that model's own `model_max_budget` is exceeded, e.g. {"gpt-4o": ["gpt-4o-mini"]}.
|
||||
* - budget_duration: Optional[str] - Budget reset period ("30d", "1h", etc.)
|
||||
* - soft_budget: Optional[float] - [TODO] Soft budget limit (warning vs. hard stop). Will trigger a slack alert when this soft budget is reached.
|
||||
|
|
@ -7032,7 +7032,7 @@ export interface paths {
|
|||
* - tags: Optional[List[str]] - Tags for organizing keys (Enterprise only)
|
||||
* - spend: Optional[float] - Amount spent by key
|
||||
* - max_budget: Optional[float] - Max budget for key
|
||||
* - model_max_budget: Optional[Dict[str, BudgetConfig]] - Model-specific budgets {"gpt-4": {"budget_limit": 0.0005, "time_period": "30d"}}
|
||||
* - model_max_budget: Optional[Dict[str, BudgetConfig]] - Model-specific budgets {"gpt-4": {"budget_limit": 0.0005, "time_period": "30d"}}. Set "models" to share one budget across a group of models: {"opus-family": {"models": ["claude-opus-4-5", "claude-opus-4-6"], "budget_limit": 10, "time_period": "30d"}}
|
||||
* - budget_fallbacks: Optional[Dict[str, List[str]]] - Per-model fallback chain tried in order when that model's own `model_max_budget` is exceeded, e.g. {"gpt-4o": ["gpt-4o-mini"]}.
|
||||
* - budget_duration: Optional[str] - Budget reset period ("30d", "1h", etc.)
|
||||
* - soft_budget: Optional[float] - Soft budget limit (warning vs. hard stop). Will trigger a slack alert when this soft budget is reached.
|
||||
|
|
@ -21395,6 +21395,8 @@ export interface components {
|
|||
budget_duration?: string | null;
|
||||
/** Max Budget */
|
||||
max_budget?: number | null;
|
||||
/** Models */
|
||||
models?: string[] | null;
|
||||
/** Rpm Limit */
|
||||
rpm_limit?: number | null;
|
||||
/** Tpm Limit */
|
||||
|
|
@ -21446,7 +21448,7 @@ export interface components {
|
|||
max_parallel_requests?: number | null;
|
||||
/**
|
||||
* Model Max Budget
|
||||
* @description Max budget for each model (e.g. {'gpt-4o': {'max_budget': '0.0000001', 'budget_duration': '1d', 'tpm_limit': 1000, 'rpm_limit': 1000}})
|
||||
* @description Max budget for each model (e.g. {'gpt-4o': {'max_budget': '0.0000001', 'budget_duration': '1d', 'tpm_limit': 1000, 'rpm_limit': 1000}}). An entry with 'models' set shares one budget across that group of models (e.g. {'opus-family': {'models': ['claude-opus-4-5', 'claude-opus-4-6'], 'max_budget': 10, 'budget_duration': '30d'}})
|
||||
*/
|
||||
model_max_budget?: {
|
||||
[key: string]: components["schemas"]["BudgetConfig"];
|
||||
|
|
@ -27759,7 +27761,7 @@ export interface components {
|
|||
max_parallel_requests?: number | null;
|
||||
/**
|
||||
* Model Max Budget
|
||||
* @description Max budget for each model (e.g. {'gpt-4o': {'max_budget': '0.0000001', 'budget_duration': '1d', 'tpm_limit': 1000, 'rpm_limit': 1000}})
|
||||
* @description Max budget for each model (e.g. {'gpt-4o': {'max_budget': '0.0000001', 'budget_duration': '1d', 'tpm_limit': 1000, 'rpm_limit': 1000}}). An entry with 'models' set shares one budget across that group of models (e.g. {'opus-family': {'models': ['claude-opus-4-5', 'claude-opus-4-6'], 'max_budget': 10, 'budget_duration': '30d'}})
|
||||
*/
|
||||
model_max_budget?: {
|
||||
[key: string]: components["schemas"]["BudgetConfig"];
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue