mirror of
https://github.com/BerriAI/litellm.git
synced 2026-08-28 05:25:59 +00:00
Merge pull request #35181 from BerriAI/litellm_block_unpriced_models
feat(proxy): add admin toggle to block requests for models without pricing
This commit is contained in:
Mateo Wang
GitHub
commit
02e67cd715
12 changed files with 878 additions and 3 deletions
|
|
@ -453,6 +453,7 @@ max_end_user_budget_id: Optional[str] = None
|
|||
# backwards compatibility — arbitrary client-supplied identifiers still
|
||||
# pass through unchanged.
|
||||
validate_end_user_id_in_db: bool = False
|
||||
block_requests_for_models_without_pricing: bool = False
|
||||
disable_end_user_cost_tracking: Optional[bool] = None
|
||||
disable_end_user_cost_tracking_prometheus_only: Optional[bool] = None
|
||||
enable_end_user_cost_tracking_prometheus_only: Optional[bool] = None
|
||||
|
|
|
|||
|
|
@ -1606,6 +1606,7 @@ LITELLM_SETTINGS_SAFE_DB_OVERRIDES: Final = [
|
|||
"public_model_groups_links",
|
||||
"cost_discount_config",
|
||||
"cost_margin_config",
|
||||
"block_requests_for_models_without_pricing",
|
||||
"budget_exceeded_throttle_percentage",
|
||||
# Every field editable from the Admin UI (proxy_server._GENERAL_SETTINGS_UI_LITELLM_FIELDS)
|
||||
# must be listed here so a DB write from one worker overrides the live litellm attribute on
|
||||
|
|
|
|||
|
|
@ -3746,6 +3746,8 @@ class ProxyErrorTypes(str, enum.Enum):
|
|||
Project does not have access to the model
|
||||
"""
|
||||
|
||||
model_cost_map_missing = "model_cost_map_missing"
|
||||
|
||||
expired_key = "expired_key"
|
||||
"""
|
||||
Key has expired
|
||||
|
|
|
|||
|
|
@ -456,6 +456,103 @@ def _is_cost_explicitly_configured(model: str, llm_router: "Router") -> bool:
|
|||
return False
|
||||
|
||||
|
||||
_EMPTY_COST_ENTRY: Final[Mapping[str, object]] = MappingProxyType({})
|
||||
|
||||
|
||||
def _is_positive_cost(value: object) -> bool:
|
||||
return isinstance(value, (int, float)) and not isinstance(value, bool) and value > 0
|
||||
|
||||
|
||||
def _entry_has_priced_metric(entry: Mapping[str, object]) -> bool:
|
||||
if entry.get("tiered_pricing") is not None:
|
||||
return True
|
||||
for key, value in entry.items():
|
||||
if "cost_per" not in key:
|
||||
continue
|
||||
if _is_positive_cost(value):
|
||||
return True
|
||||
if isinstance(value, dict) and any(_is_positive_cost(nested) for nested in value.values()):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _entry_declares_price(entry: Mapping[str, object]) -> bool:
|
||||
return any("cost_per" in key or key == "tiered_pricing" for key in entry)
|
||||
|
||||
|
||||
def _model_group_has_pricing(model: str, llm_router: "Router") -> bool:
|
||||
"""
|
||||
A model group counts as priced when a deployment overrides any *cost_per* field or
|
||||
tiered_pricing in its litellm_params, even at zero, or when its resolved model info carries
|
||||
tiered_pricing or a positive price on any billed metric (tokens, characters, seconds, pages,
|
||||
images, queries, ...), so models billed by a non-token metric are not treated as unpriced.
|
||||
"""
|
||||
for deployment in llm_router.get_model_list(model_name=model) or ():
|
||||
litellm_params = deployment.get("litellm_params") or _EMPTY_COST_ENTRY
|
||||
if _entry_declares_price(litellm_params):
|
||||
return True
|
||||
|
||||
model_id = (deployment.get("model_info") or _EMPTY_COST_ENTRY).get("id")
|
||||
if model_id is None:
|
||||
continue
|
||||
|
||||
model_info = llm_router.get_deployment_model_info(
|
||||
model_id=model_id, model_name=litellm_params.get("model") or ""
|
||||
)
|
||||
if model_info is not None and _entry_has_priced_metric(model_info):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def _group_declares_explicit_cost(model: str, llm_router: "Router") -> bool:
|
||||
"""
|
||||
Alias-aware counterpart to ``_is_cost_explicitly_configured``, which resolves the model group
|
||||
the same way ``_model_group_has_pricing`` does. A deployment that prices itself through its
|
||||
``model_info`` block lands in the cost map under its deployment id rather than in its
|
||||
litellm_params, and reaching that entry through the router's own resolution keeps an alias
|
||||
pointing at such a group from being read as unpriced.
|
||||
"""
|
||||
for deployment in llm_router.get_model_list(model_name=model) or ():
|
||||
model_id = (deployment.get("model_info") or _EMPTY_COST_ENTRY).get("id")
|
||||
if model_id is None:
|
||||
continue
|
||||
raw_entry = litellm.model_cost.get(model_id, _EMPTY_COST_ENTRY)
|
||||
if "input_cost_per_token" in raw_entry or "output_cost_per_token" in raw_entry:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def model_has_no_cost_mapping(model: str | None, llm_router: Router | None) -> bool:
|
||||
if not model or llm_router is None:
|
||||
return False
|
||||
|
||||
if llm_router.get_model_group_info(model_group=model) is None:
|
||||
return False
|
||||
|
||||
if _model_group_has_pricing(model=model, llm_router=llm_router):
|
||||
return False
|
||||
|
||||
return not _group_declares_explicit_cost(model=model, llm_router=llm_router)
|
||||
|
||||
|
||||
def _unpriced_models_in_request(model: str | list[str] | None, llm_router: Router | None) -> tuple[str, ...]:
|
||||
candidates: Final = (model,) if isinstance(model, str) else tuple(model or ())
|
||||
return tuple(
|
||||
candidate for candidate in candidates if model_has_no_cost_mapping(model=candidate, llm_router=llm_router)
|
||||
)
|
||||
|
||||
|
||||
def _unpriced_models_block_message(models: tuple[str, ...]) -> str:
|
||||
names: Final = ", ".join(f"'{model}'" for model in models)
|
||||
subject: Final = f"Model {names} has" if len(models) == 1 else f"Models {names} have"
|
||||
return (
|
||||
f"{subject} no pricing in the cost map, so litellm cannot price the request. "
|
||||
"Requests for unpriced models are blocked because 'block_requests_for_models_without_pricing' "
|
||||
"is enabled. Add pricing (input_cost_per_token/output_cost_per_token) to allow the request."
|
||||
)
|
||||
|
||||
|
||||
async def _run_project_checks(
|
||||
project_object: LiteLLM_ProjectTableCachedObj | None,
|
||||
_model: str | list[str] | None,
|
||||
|
|
@ -726,6 +823,19 @@ async def common_checks(
|
|||
and (route in MODEL_DISCOVERY_ROUTES or not RouteChecks.is_llm_api_route(route=route))
|
||||
)
|
||||
|
||||
unpriced_models: Final = (
|
||||
_unpriced_models_in_request(model=_model, llm_router=llm_router)
|
||||
if litellm.block_requests_for_models_without_pricing and RouteChecks.is_llm_api_route(route=route)
|
||||
else ()
|
||||
)
|
||||
if unpriced_models:
|
||||
raise ProxyException(
|
||||
message=_unpriced_models_block_message(unpriced_models),
|
||||
type=ProxyErrorTypes.model_cost_map_missing,
|
||||
param="model",
|
||||
code=status.HTTP_403_FORBIDDEN,
|
||||
)
|
||||
|
||||
# 1. If team is blocked
|
||||
if team_object is not None and team_object.blocked is True:
|
||||
raise Exception(f"Team={team_object.team_id} is blocked. Update via `/team/unblock` if you're an admin.")
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ from dataclasses import dataclass
|
|||
from typing import Final
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from pydantic import BaseModel
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
|
|
@ -439,6 +440,76 @@ async def update_cost_margin_config(
|
|||
)
|
||||
|
||||
|
||||
class BlockUnpricedModelsRequest(BaseModel):
|
||||
enabled: bool
|
||||
|
||||
|
||||
class BlockUnpricedModelsResponse(BaseModel):
|
||||
enabled: bool
|
||||
|
||||
|
||||
@router.get(
|
||||
"/config/block_requests_for_models_without_pricing",
|
||||
tags=("Cost Tracking",),
|
||||
dependencies=(Depends(user_api_key_auth),),
|
||||
response_model=BlockUnpricedModelsResponse,
|
||||
)
|
||||
async def get_block_requests_for_models_without_pricing() -> BlockUnpricedModelsResponse:
|
||||
return BlockUnpricedModelsResponse(enabled=bool(litellm.block_requests_for_models_without_pricing))
|
||||
|
||||
|
||||
@router.patch(
|
||||
"/config/block_requests_for_models_without_pricing",
|
||||
tags=("Cost Tracking",),
|
||||
dependencies=(Depends(user_api_key_auth),),
|
||||
response_model=BlockUnpricedModelsResponse,
|
||||
)
|
||||
async def update_block_requests_for_models_without_pricing(
|
||||
request: BlockUnpricedModelsRequest,
|
||||
) -> BlockUnpricedModelsResponse:
|
||||
from litellm.proxy.proxy_server import (
|
||||
prisma_client,
|
||||
proxy_config,
|
||||
store_model_in_db,
|
||||
)
|
||||
|
||||
if prisma_client is None:
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail={ # mutable-ok: HTTPException detail must be a plain mapping
|
||||
"error": CommonProxyErrors.db_not_connected_error.value
|
||||
},
|
||||
)
|
||||
|
||||
if store_model_in_db is not True:
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail={ # mutable-ok: HTTPException detail must be a plain mapping
|
||||
"error": "Set `'STORE_MODEL_IN_DB='True'` in your env to enable this feature."
|
||||
},
|
||||
)
|
||||
|
||||
try:
|
||||
config = await proxy_config.get_config()
|
||||
if "litellm_settings" not in config:
|
||||
config["litellm_settings"] = {} # mutable-ok: config is a plain-dict payload for save_config
|
||||
config["litellm_settings"]["block_requests_for_models_without_pricing"] = request.enabled
|
||||
await proxy_config.save_config(new_config=config)
|
||||
|
||||
litellm.block_requests_for_models_without_pricing = request.enabled
|
||||
verbose_proxy_logger.info("Updated block_requests_for_models_without_pricing: %s", request.enabled)
|
||||
|
||||
return BlockUnpricedModelsResponse(enabled=request.enabled)
|
||||
except Exception as e: # noqa: BLE001 # any config persistence failure must surface as a 500 response, not a crash
|
||||
verbose_proxy_logger.error("Error updating block_requests_for_models_without_pricing: %s", e)
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail={ # mutable-ok: HTTPException detail must be a plain mapping
|
||||
"error": f"Failed to update setting: {e!s}"
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/cost/estimate",
|
||||
tags=["Cost Tracking"],
|
||||
|
|
|
|||
|
|
@ -6840,6 +6840,20 @@ class ProxyConfig:
|
|||
if self._should_load_db_object(object_type="config_overrides"):
|
||||
await self._init_hashicorp_vault_config_override(prisma_client=prisma_client)
|
||||
|
||||
await self._apply_safe_litellm_settings_overrides_from_db(prisma_client=prisma_client)
|
||||
|
||||
async def _apply_safe_litellm_settings_overrides_from_db(self, prisma_client: PrismaClient) -> None:
|
||||
config_record: Final = await get_config_param(prisma_client, "litellm_settings")
|
||||
if config_record is None or config_record.param_value is None:
|
||||
return
|
||||
raw_settings: Final = config_record.param_value
|
||||
litellm_settings: Final = json.loads(raw_settings) if isinstance(raw_settings, str) else raw_settings
|
||||
if not isinstance(litellm_settings, dict):
|
||||
return
|
||||
for key, value in litellm_settings.items():
|
||||
if key in LITELLM_SETTINGS_SAFE_DB_OVERRIDES:
|
||||
setattr(litellm, key, value)
|
||||
|
||||
async def _init_semantic_filter_settings_in_db(self, prisma_client: PrismaClient):
|
||||
"""
|
||||
Initialize MCP semantic filter settings from database.
|
||||
|
|
|
|||
|
|
@ -3,8 +3,12 @@ import json
|
|||
import os
|
||||
import sys
|
||||
from types import SimpleNamespace
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.router import Router
|
||||
|
||||
sys.path.insert(
|
||||
0, os.path.abspath("../../..")
|
||||
) # Adds the parent directory to the system path
|
||||
|
|
@ -6612,3 +6616,284 @@ def test_can_object_call_model_team_scoped_wildcard_accepts_bare_model_name():
|
|||
)
|
||||
is True
|
||||
)
|
||||
|
||||
|
||||
UNPRICED_UNDERLYING_MODEL = "openai/unpriced-model-lit4984-xyz"
|
||||
|
||||
|
||||
def _router_with_priced_and_unpriced_models() -> "Router":
|
||||
from litellm.router import Router
|
||||
|
||||
return Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "priced-group",
|
||||
"litellm_params": {"model": "gpt-3.5-turbo", "api_key": "sk-test"},
|
||||
},
|
||||
{
|
||||
"model_name": "unpriced-group",
|
||||
"litellm_params": {"model": UNPRICED_UNDERLYING_MODEL, "api_key": "sk-test"},
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def test_model_has_no_cost_mapping_priced_model_is_false():
|
||||
from litellm.proxy.auth.auth_checks import model_has_no_cost_mapping
|
||||
|
||||
router = _router_with_priced_and_unpriced_models()
|
||||
|
||||
assert model_has_no_cost_mapping(model="priced-group", llm_router=router) is False
|
||||
|
||||
|
||||
def test_model_has_no_cost_mapping_unpriced_model_is_true():
|
||||
from litellm.proxy.auth.auth_checks import model_has_no_cost_mapping
|
||||
|
||||
router = _router_with_priced_and_unpriced_models()
|
||||
|
||||
assert model_has_no_cost_mapping(model="unpriced-group", llm_router=router) is True
|
||||
|
||||
|
||||
def test_model_has_no_cost_mapping_no_model_or_router_is_false():
|
||||
from litellm.proxy.auth.auth_checks import model_has_no_cost_mapping
|
||||
|
||||
router = _router_with_priced_and_unpriced_models()
|
||||
|
||||
assert model_has_no_cost_mapping(model=None, llm_router=router) is False
|
||||
assert model_has_no_cost_mapping(model="unpriced-group", llm_router=None) is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"underlying_model",
|
||||
[
|
||||
"azure/speech/azure-tts",
|
||||
"mistral/mistral-ocr-latest",
|
||||
"vertex_ai/imagen-3.0-generate-001",
|
||||
"dashscope/qwen-flash",
|
||||
],
|
||||
)
|
||||
def test_model_has_no_cost_mapping_non_token_priced_model_is_false(underlying_model):
|
||||
from litellm.proxy.auth.auth_checks import model_has_no_cost_mapping
|
||||
from litellm.router import Router
|
||||
|
||||
router = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "non-token-priced-group",
|
||||
"litellm_params": {"model": underlying_model, "api_key": "sk-test"},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
assert model_has_no_cost_mapping(model="non-token-priced-group", llm_router=router) is False
|
||||
|
||||
|
||||
def test_model_has_no_cost_mapping_non_token_price_from_litellm_params_is_false():
|
||||
from litellm.proxy.auth.auth_checks import model_has_no_cost_mapping
|
||||
from litellm.router import Router
|
||||
|
||||
router = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "custom-tts",
|
||||
"litellm_params": {
|
||||
"model": f"{UNPRICED_UNDERLYING_MODEL}-per-second",
|
||||
"api_key": "sk-test",
|
||||
"input_cost_per_second": 0.0001,
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
assert model_has_no_cost_mapping(model="custom-tts", llm_router=router) is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cost_field", ["input_cost_per_second", "input_cost_per_token"])
|
||||
def test_model_has_no_cost_mapping_explicit_zero_price_is_false(cost_field):
|
||||
from litellm.proxy.auth.auth_checks import model_has_no_cost_mapping
|
||||
from litellm.router import Router
|
||||
|
||||
router = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "free-group",
|
||||
"litellm_params": {
|
||||
"model": f"{UNPRICED_UNDERLYING_MODEL}-{cost_field}",
|
||||
"api_key": "sk-test",
|
||||
cost_field: 0,
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
assert model_has_no_cost_mapping(model="free-group", llm_router=router) is False
|
||||
|
||||
|
||||
def test_model_has_no_cost_mapping_tiered_pricing_only_is_false():
|
||||
from litellm.proxy.auth.auth_checks import model_has_no_cost_mapping
|
||||
from litellm.router import Router
|
||||
|
||||
router = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "tiered-group",
|
||||
"litellm_params": {
|
||||
"model": f"{UNPRICED_UNDERLYING_MODEL}-tiered",
|
||||
"api_key": "sk-test",
|
||||
"tiered_pricing": [
|
||||
{"range": [0, 128000], "input_cost_per_token": 2e-7, "output_cost_per_token": 6e-7},
|
||||
{"range": [128000, 256000], "input_cost_per_token": 4e-7, "output_cost_per_token": 12e-7},
|
||||
],
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
assert model_has_no_cost_mapping(model="tiered-group", llm_router=router) is False
|
||||
|
||||
|
||||
async def _run_common_checks(
|
||||
model: Optional[str], llm_router: Optional["Router"], route: str = "/chat/completions"
|
||||
) -> bool:
|
||||
from fastapi import Request
|
||||
|
||||
from litellm.proxy.auth.auth_checks import common_checks
|
||||
|
||||
return await common_checks(
|
||||
request_body={"model": model, "messages": [{"role": "user", "content": "hi"}]},
|
||||
team_object=None,
|
||||
user_object=None,
|
||||
end_user_object=None,
|
||||
global_proxy_spend=None,
|
||||
general_settings={},
|
||||
route=route,
|
||||
llm_router=llm_router,
|
||||
proxy_logging_obj=MagicMock(),
|
||||
valid_token=UserAPIKeyAuth(token="test-token"),
|
||||
request=MagicMock(spec=Request),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_common_checks_blocks_unpriced_model_when_enabled(monkeypatch):
|
||||
monkeypatch.setattr(litellm, "block_requests_for_models_without_pricing", True)
|
||||
router = _router_with_priced_and_unpriced_models()
|
||||
|
||||
with pytest.raises(ProxyException) as exc_info:
|
||||
await _run_common_checks(model="unpriced-group", llm_router=router)
|
||||
|
||||
assert exc_info.value.code == "403"
|
||||
assert exc_info.value.type == ProxyErrorTypes.model_cost_map_missing
|
||||
assert exc_info.value.param == "model"
|
||||
assert "unpriced-group" in exc_info.value.message
|
||||
assert "pricing" in exc_info.value.message.lower()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_common_checks_allows_unpriced_model_when_disabled(monkeypatch):
|
||||
monkeypatch.setattr(litellm, "block_requests_for_models_without_pricing", False)
|
||||
router = _router_with_priced_and_unpriced_models()
|
||||
|
||||
result = await _run_common_checks(model="unpriced-group", llm_router=router)
|
||||
|
||||
assert result is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_common_checks_allows_priced_model_when_enabled(monkeypatch):
|
||||
monkeypatch.setattr(litellm, "block_requests_for_models_without_pricing", True)
|
||||
router = _router_with_priced_and_unpriced_models()
|
||||
|
||||
result = await _run_common_checks(model="priced-group", llm_router=router)
|
||||
|
||||
assert result is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_common_checks_ignores_non_llm_route_when_enabled(monkeypatch):
|
||||
monkeypatch.setattr(litellm, "block_requests_for_models_without_pricing", True)
|
||||
router = _router_with_priced_and_unpriced_models()
|
||||
|
||||
result = await _run_common_checks(
|
||||
model="unpriced-group", llm_router=router, route="/model/new"
|
||||
)
|
||||
|
||||
assert result is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_common_checks_blocks_alias_resolving_to_unpriced_model(monkeypatch):
|
||||
from litellm.router import Router
|
||||
|
||||
monkeypatch.setattr(litellm, "block_requests_for_models_without_pricing", True)
|
||||
router = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "billed-underlying-group",
|
||||
"litellm_params": {"model": UNPRICED_UNDERLYING_MODEL, "api_key": "sk-test"},
|
||||
}
|
||||
],
|
||||
model_group_alias={"public-alias": "billed-underlying-group"},
|
||||
)
|
||||
|
||||
with pytest.raises(ProxyException) as exc_info:
|
||||
await _run_common_checks(model="public-alias", llm_router=router)
|
||||
|
||||
assert exc_info.value.code == "403"
|
||||
assert exc_info.value.type == ProxyErrorTypes.model_cost_map_missing
|
||||
assert "public-alias" in exc_info.value.message
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_common_checks_blocks_comma_separated_request_carrying_an_unpriced_model(monkeypatch):
|
||||
monkeypatch.setattr(litellm, "block_requests_for_models_without_pricing", True)
|
||||
router = _router_with_priced_and_unpriced_models()
|
||||
|
||||
with pytest.raises(ProxyException) as exc_info:
|
||||
await _run_common_checks(model="priced-group,unpriced-group", llm_router=router)
|
||||
|
||||
assert exc_info.value.code == "403"
|
||||
assert exc_info.value.type == ProxyErrorTypes.model_cost_map_missing
|
||||
assert "'unpriced-group'" in exc_info.value.message
|
||||
assert "'priced-group'" not in exc_info.value.message
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_common_checks_allows_comma_separated_request_when_every_model_is_priced(monkeypatch):
|
||||
monkeypatch.setattr(litellm, "block_requests_for_models_without_pricing", True)
|
||||
router = _router_with_priced_and_unpriced_models()
|
||||
|
||||
result = await _run_common_checks(model="priced-group,priced-group", llm_router=router)
|
||||
|
||||
assert result is True
|
||||
|
||||
|
||||
def _router_with_a_group_priced_through_model_info() -> "Router":
|
||||
from litellm.router import Router
|
||||
|
||||
return Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "model-info-priced-group",
|
||||
"litellm_params": {"model": f"{UNPRICED_UNDERLYING_MODEL}-model-info", "api_key": "sk-test"},
|
||||
"model_info": {"input_cost_per_token": 0, "output_cost_per_token": 0},
|
||||
}
|
||||
],
|
||||
model_group_alias={"model-info-priced-alias": "model-info-priced-group"},
|
||||
)
|
||||
|
||||
|
||||
def test_model_has_no_cost_mapping_group_priced_through_model_info_is_false():
|
||||
from litellm.proxy.auth.auth_checks import model_has_no_cost_mapping
|
||||
|
||||
router = _router_with_a_group_priced_through_model_info()
|
||||
|
||||
assert model_has_no_cost_mapping(model="model-info-priced-group", llm_router=router) is False
|
||||
|
||||
|
||||
def test_model_has_no_cost_mapping_alias_to_a_group_priced_through_model_info_is_false():
|
||||
from litellm.proxy.auth.auth_checks import model_has_no_cost_mapping
|
||||
|
||||
router = _router_with_a_group_priced_through_model_info()
|
||||
|
||||
assert model_has_no_cost_mapping(model="model-info-priced-alias", llm_router=router) is False
|
||||
|
|
|
|||
|
|
@ -683,3 +683,103 @@ class TestEstimateCostOnPremProvider:
|
|||
assert response.cost_per_request == pytest.approx(0.002)
|
||||
assert response.input_cost_per_token == pytest.approx(0.000001)
|
||||
assert response.output_cost_per_token == pytest.approx(0.000002)
|
||||
|
||||
|
||||
|
||||
|
||||
class TestBlockRequestsForModelsWithoutPricing:
|
||||
"""Test suite for the block_requests_for_models_without_pricing toggle endpoints"""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_reflects_in_memory_flag(self):
|
||||
with patch.object(litellm, "block_requests_for_models_without_pricing", True):
|
||||
response = client.get(
|
||||
"/config/block_requests_for_models_without_pricing",
|
||||
headers={"Authorization": "Bearer sk-1234"},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json() == {"enabled": True}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_patch_persists_and_updates_flag(self):
|
||||
mock_proxy_config = AsyncMock()
|
||||
mock_proxy_config.get_config = AsyncMock(return_value={"litellm_settings": {}})
|
||||
mock_proxy_config.save_config = AsyncMock()
|
||||
|
||||
with (
|
||||
patch("litellm.proxy.proxy_server.prisma_client", MagicMock()),
|
||||
patch("litellm.proxy.proxy_server.proxy_config", mock_proxy_config),
|
||||
patch("litellm.proxy.proxy_server.store_model_in_db", True),
|
||||
patch.object(litellm, "block_requests_for_models_without_pricing", False),
|
||||
):
|
||||
response = client.patch(
|
||||
"/config/block_requests_for_models_without_pricing",
|
||||
headers={"Authorization": "Bearer sk-1234"},
|
||||
json={"enabled": True},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json() == {"enabled": True}
|
||||
assert litellm.block_requests_for_models_without_pricing is True
|
||||
|
||||
saved_config = mock_proxy_config.save_config.call_args.kwargs["new_config"]
|
||||
assert saved_config["litellm_settings"]["block_requests_for_models_without_pricing"] is True
|
||||
|
||||
def test_peer_workers_pick_up_persisted_flag_on_config_reload(self):
|
||||
"""A PATCH only mutates the flag on the worker that served it; peer workers must pick the
|
||||
persisted value up when they reload litellm_settings from the DB."""
|
||||
from litellm.proxy.proxy_server import ProxyConfig
|
||||
|
||||
with patch.object(litellm, "block_requests_for_models_without_pricing", False):
|
||||
ProxyConfig()._update_config_fields(
|
||||
current_config={},
|
||||
param_name="litellm_settings",
|
||||
db_param_value={"block_requests_for_models_without_pricing": True},
|
||||
)
|
||||
|
||||
assert litellm.block_requests_for_models_without_pricing is True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("loads_config_overrides", [True, False])
|
||||
async def test_periodic_db_sync_applies_flag_to_peer_worker(self, loads_config_overrides):
|
||||
"""The ~10s reconcile loop runs _init_non_llm_objects_in_db on every worker; it must apply
|
||||
the persisted flag so peers converge without a restart, including when supported_db_objects
|
||||
leaves config_overrides out."""
|
||||
from types import SimpleNamespace
|
||||
|
||||
from litellm.proxy.proxy_server import ProxyConfig
|
||||
|
||||
config_record = SimpleNamespace(
|
||||
param_value={"block_requests_for_models_without_pricing": True, "unsafe_key": "x"}
|
||||
)
|
||||
with (
|
||||
patch.object(litellm, "block_requests_for_models_without_pricing", False),
|
||||
patch.object(
|
||||
ProxyConfig,
|
||||
"_should_load_db_object",
|
||||
side_effect=lambda object_type: loads_config_overrides and object_type == "config_overrides",
|
||||
),
|
||||
patch.object(ProxyConfig, "_init_hashicorp_vault_config_override", AsyncMock()),
|
||||
patch("litellm.proxy.proxy_server.get_config_param", AsyncMock(return_value=config_record)),
|
||||
):
|
||||
await ProxyConfig()._init_non_llm_objects_in_db(prisma_client=MagicMock())
|
||||
|
||||
assert litellm.block_requests_for_models_without_pricing is True
|
||||
assert not hasattr(litellm, "unsafe_key")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_patch_requires_store_model_in_db(self):
|
||||
with (
|
||||
patch("litellm.proxy.proxy_server.prisma_client", MagicMock()),
|
||||
patch("litellm.proxy.proxy_server.proxy_config", AsyncMock()),
|
||||
patch("litellm.proxy.proxy_server.store_model_in_db", False),
|
||||
):
|
||||
response = client.patch(
|
||||
"/config/block_requests_for_models_without_pricing",
|
||||
headers={"Authorization": "Bearer sk-1234"},
|
||||
json={"enabled": True},
|
||||
)
|
||||
|
||||
assert response.status_code == 500
|
||||
assert "error" in response.json()["detail"]
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ import {
|
|||
} from "@/components/ui/alert-dialog";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible";
|
||||
import { Switch } from "@/components/ui/switch";
|
||||
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
|
||||
import { CostTrackingSettingsProps } from "./types";
|
||||
import ProviderDiscountTable from "./provider_discount_table";
|
||||
|
|
@ -22,6 +23,7 @@ import { DocsMenu } from "@/components/HelpLink";
|
|||
import HowItWorks from "./how_it_works";
|
||||
import { useDiscountConfig } from "./use_discount_config";
|
||||
import { useMarginConfig } from "./use_margin_config";
|
||||
import { useBlockUnpricedConfig } from "./use_block_unpriced_config";
|
||||
import { fetchAvailableModels, ModelGroup } from "@/components/llm_calls/fetch_models";
|
||||
import { Dialog, DialogContent, DialogHeader, DialogTitle } from "@/components/ui/dialog";
|
||||
|
||||
|
|
@ -86,9 +88,16 @@ const CostTrackingSettings: React.FC<CostTrackingSettingsProps> = ({ userID, use
|
|||
handleMarginChange,
|
||||
} = useMarginConfig({ accessToken });
|
||||
|
||||
const {
|
||||
blockUnpriced,
|
||||
isUpdating: isUpdatingBlockUnpriced,
|
||||
fetchBlockUnpriced,
|
||||
setBlockUnpriced,
|
||||
} = useBlockUnpricedConfig({ accessToken });
|
||||
|
||||
useEffect(() => {
|
||||
if (accessToken) {
|
||||
Promise.all([fetchDiscountConfig(), fetchMarginConfig()]).finally(() => {
|
||||
Promise.all([fetchDiscountConfig(), fetchMarginConfig(), fetchBlockUnpriced()]).finally(() => {
|
||||
setIsFetching(false);
|
||||
});
|
||||
|
||||
|
|
@ -103,7 +112,7 @@ const CostTrackingSettings: React.FC<CostTrackingSettingsProps> = ({ userID, use
|
|||
};
|
||||
loadModels();
|
||||
}
|
||||
}, [accessToken, fetchDiscountConfig, fetchMarginConfig]);
|
||||
}, [accessToken, fetchDiscountConfig, fetchMarginConfig, fetchBlockUnpriced]);
|
||||
|
||||
const handleAddProvider = async () => {
|
||||
const success = await addProvider(selectedProvider, newDiscount);
|
||||
|
|
@ -301,7 +310,35 @@ const CostTrackingSettings: React.FC<CostTrackingSettingsProps> = ({ userID, use
|
|||
</Collapsible>
|
||||
)}
|
||||
|
||||
{/* Accordion 3: Pricing Calculator - Available to all roles */}
|
||||
{/* Accordion 3: Block Unpriced Models - Only for proxy admins */}
|
||||
{isProxyAdmin && (
|
||||
<Collapsible className="rounded-lg border">
|
||||
<SectionHeader
|
||||
title="Block Unpriced Models"
|
||||
description="Reject requests for models that have no pricing in the cost map instead of logging them as $0 spend"
|
||||
/>
|
||||
<CollapsibleContent className="px-0">
|
||||
<div className="p-6">
|
||||
<div className="flex items-center justify-between">
|
||||
<div className="pr-6">
|
||||
<p className="text-foreground font-medium">Block requests for models without pricing</p>
|
||||
<p className="text-sm text-muted-foreground mt-1">
|
||||
When enabled, a request whose resolved model has no cost mapping is rejected with a 403 so an
|
||||
admin can add pricing for it. Off by default
|
||||
</p>
|
||||
</div>
|
||||
<Switch
|
||||
checked={blockUnpriced}
|
||||
disabled={isUpdatingBlockUnpriced || isFetching}
|
||||
onCheckedChange={(checked) => setBlockUnpriced(checked)}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</CollapsibleContent>
|
||||
</Collapsible>
|
||||
)}
|
||||
|
||||
{/* Accordion 4: Pricing Calculator - Available to all roles */}
|
||||
<Collapsible defaultOpen={true} className="rounded-lg border">
|
||||
<SectionHeader
|
||||
title="Pricing Calculator"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,108 @@
|
|||
import { describe, it, expect, vi, beforeEach } from "vitest";
|
||||
import { renderHook, act } from "@testing-library/react";
|
||||
import { useBlockUnpricedConfig } from "./use_block_unpriced_config";
|
||||
import { apiClient } from "@/components/networking";
|
||||
import { toast } from "@/lib/toast";
|
||||
|
||||
vi.mock("@/components/networking", () => ({
|
||||
apiClient: {
|
||||
get: vi.fn(),
|
||||
patch: vi.fn(),
|
||||
},
|
||||
}));
|
||||
|
||||
const ENDPOINT = "/config/block_requests_for_models_without_pricing";
|
||||
|
||||
describe("useBlockUnpricedConfig", () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
describe("fetchBlockUnpriced", () => {
|
||||
it("reflects the enabled flag returned by the proxy", async () => {
|
||||
vi.mocked(apiClient.get).mockResolvedValueOnce({ enabled: true });
|
||||
|
||||
const { result } = renderHook(() => useBlockUnpricedConfig({ accessToken: "test-token" }));
|
||||
|
||||
await act(async () => {
|
||||
await result.current.fetchBlockUnpriced();
|
||||
});
|
||||
|
||||
expect(apiClient.get).toHaveBeenCalledWith(ENDPOINT, { accessToken: "test-token" });
|
||||
expect(result.current.blockUnpriced).toBe(true);
|
||||
});
|
||||
|
||||
it("surfaces a toast when the fetch throws", async () => {
|
||||
const error = new Error("Network error");
|
||||
vi.mocked(apiClient.get).mockRejectedValueOnce(error);
|
||||
|
||||
const { result } = renderHook(() => useBlockUnpricedConfig({ accessToken: "test-token" }));
|
||||
|
||||
await act(async () => {
|
||||
await result.current.fetchBlockUnpriced();
|
||||
});
|
||||
|
||||
expect(toast.fromError).toHaveBeenCalledWith(error);
|
||||
expect(result.current.blockUnpriced).toBe(false);
|
||||
});
|
||||
|
||||
it("does nothing without an access token", async () => {
|
||||
const { result } = renderHook(() => useBlockUnpricedConfig({ accessToken: null }));
|
||||
|
||||
await act(async () => {
|
||||
await result.current.fetchBlockUnpriced();
|
||||
});
|
||||
|
||||
expect(apiClient.get).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe("setBlockUnpriced", () => {
|
||||
it("persists the new value and confirms it with a toast", async () => {
|
||||
vi.mocked(apiClient.patch).mockResolvedValueOnce({ enabled: true });
|
||||
|
||||
const { result } = renderHook(() => useBlockUnpricedConfig({ accessToken: "test-token" }));
|
||||
|
||||
await act(async () => {
|
||||
await result.current.setBlockUnpriced(true);
|
||||
});
|
||||
|
||||
expect(apiClient.patch).toHaveBeenCalledWith(ENDPOINT, {
|
||||
accessToken: "test-token",
|
||||
body: { enabled: true },
|
||||
});
|
||||
expect(result.current.blockUnpriced).toBe(true);
|
||||
expect(toast.success).toHaveBeenCalledWith(expect.stringMatching(/will now be blocked/i));
|
||||
expect(result.current.isUpdating).toBe(false);
|
||||
});
|
||||
|
||||
it("confirms turning the block back off", async () => {
|
||||
vi.mocked(apiClient.patch).mockResolvedValueOnce({ enabled: false });
|
||||
|
||||
const { result } = renderHook(() => useBlockUnpricedConfig({ accessToken: "test-token" }));
|
||||
|
||||
await act(async () => {
|
||||
await result.current.setBlockUnpriced(false);
|
||||
});
|
||||
|
||||
expect(result.current.blockUnpriced).toBe(false);
|
||||
expect(toast.success).toHaveBeenCalledWith(expect.stringMatching(/now allowed/i));
|
||||
});
|
||||
|
||||
it("surfaces the proxy error and leaves the flag unchanged when the update fails", async () => {
|
||||
const error = new Error("Set `'STORE_MODEL_IN_DB='True'` in your env to enable this feature.");
|
||||
vi.mocked(apiClient.patch).mockRejectedValueOnce(error);
|
||||
|
||||
const { result } = renderHook(() => useBlockUnpricedConfig({ accessToken: "test-token" }));
|
||||
|
||||
await act(async () => {
|
||||
await result.current.setBlockUnpriced(true);
|
||||
});
|
||||
|
||||
expect(toast.fromError).toHaveBeenCalledWith(error);
|
||||
expect(toast.success).not.toHaveBeenCalled();
|
||||
expect(result.current.blockUnpriced).toBe(false);
|
||||
expect(result.current.isUpdating).toBe(false);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,65 @@
|
|||
import { useState, useCallback } from "react";
|
||||
import { apiClient } from "@/components/networking";
|
||||
import { toast } from "@/lib/toast";
|
||||
|
||||
export interface UseBlockUnpricedConfigProps {
|
||||
accessToken: string | null;
|
||||
}
|
||||
|
||||
export interface UseBlockUnpricedConfigReturn {
|
||||
blockUnpriced: boolean;
|
||||
isUpdating: boolean;
|
||||
fetchBlockUnpriced: () => Promise<void>;
|
||||
setBlockUnpriced: (enabled: boolean) => Promise<void>;
|
||||
}
|
||||
|
||||
interface BlockUnpricedResponse {
|
||||
enabled: boolean;
|
||||
}
|
||||
|
||||
const ENDPOINT = "/config/block_requests_for_models_without_pricing";
|
||||
|
||||
export function useBlockUnpricedConfig({ accessToken }: UseBlockUnpricedConfigProps): UseBlockUnpricedConfigReturn {
|
||||
const [blockUnpriced, setBlockUnpricedState] = useState<boolean>(false);
|
||||
const [isUpdating, setIsUpdating] = useState<boolean>(false);
|
||||
|
||||
const fetchBlockUnpriced = useCallback(async () => {
|
||||
if (!accessToken) return;
|
||||
try {
|
||||
const data = await apiClient.get<BlockUnpricedResponse>(ENDPOINT, { accessToken });
|
||||
setBlockUnpricedState(Boolean(data?.enabled));
|
||||
} catch (error) {
|
||||
console.error("Error fetching block-unpriced-models setting:", error);
|
||||
toast.fromError(error);
|
||||
}
|
||||
}, [accessToken]);
|
||||
|
||||
const setBlockUnpriced = useCallback(
|
||||
async (enabled: boolean) => {
|
||||
if (!accessToken) return;
|
||||
setIsUpdating(true);
|
||||
try {
|
||||
const data = await apiClient.patch<BlockUnpricedResponse>(ENDPOINT, { accessToken, body: { enabled } });
|
||||
setBlockUnpricedState(Boolean(data?.enabled));
|
||||
toast.success(
|
||||
enabled
|
||||
? "Requests for models without pricing will now be blocked"
|
||||
: "Requests for models without pricing are now allowed",
|
||||
);
|
||||
} catch (error) {
|
||||
console.error("Error updating block-unpriced-models setting:", error);
|
||||
toast.fromError(error);
|
||||
} finally {
|
||||
setIsUpdating(false);
|
||||
}
|
||||
},
|
||||
[accessToken],
|
||||
);
|
||||
|
||||
return {
|
||||
blockUnpriced,
|
||||
isUpdating,
|
||||
fetchBlockUnpriced,
|
||||
setBlockUnpriced,
|
||||
};
|
||||
}
|
||||
81
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
81
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -2146,6 +2146,24 @@ export interface paths {
|
|||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/config/block_requests_for_models_without_pricing": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
/** Get Block Requests For Models Without Pricing */
|
||||
get: operations["get_block_requests_for_models_without_pricing_config_block_requests_for_models_without_pricing_get"];
|
||||
put?: never;
|
||||
post?: never;
|
||||
delete?: never;
|
||||
options?: never;
|
||||
head?: never;
|
||||
/** Update Block Requests For Models Without Pricing */
|
||||
patch: operations["update_block_requests_for_models_without_pricing_config_block_requests_for_models_without_pricing_patch"];
|
||||
trace?: never;
|
||||
};
|
||||
"/config/callback/delete": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
|
|
@ -22457,6 +22475,16 @@ export interface components {
|
|||
/** Team Id */
|
||||
team_id: string;
|
||||
};
|
||||
/** BlockUnpricedModelsRequest */
|
||||
BlockUnpricedModelsRequest: {
|
||||
/** Enabled */
|
||||
enabled: boolean;
|
||||
};
|
||||
/** BlockUnpricedModelsResponse */
|
||||
BlockUnpricedModelsResponse: {
|
||||
/** Enabled */
|
||||
enabled: boolean;
|
||||
};
|
||||
/** BlockUsers */
|
||||
BlockUsers: {
|
||||
/** User Ids */
|
||||
|
|
@ -40173,6 +40201,59 @@ export interface operations {
|
|||
};
|
||||
};
|
||||
};
|
||||
get_block_requests_for_models_without_pricing_config_block_requests_for_models_without_pricing_get: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
requestBody?: never;
|
||||
responses: {
|
||||
/** @description Successful Response */
|
||||
200: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["BlockUnpricedModelsResponse"];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
update_block_requests_for_models_without_pricing_config_block_requests_for_models_without_pricing_patch: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
requestBody: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["BlockUnpricedModelsRequest"];
|
||||
};
|
||||
};
|
||||
responses: {
|
||||
/** @description Successful Response */
|
||||
200: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["BlockUnpricedModelsResponse"];
|
||||
};
|
||||
};
|
||||
/** @description Validation Error */
|
||||
422: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["HTTPValidationError"];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
delete_callback_config_callback_delete_post: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue