mirror of
https://github.com/BerriAI/litellm.git
synced 2026-08-28 05:25:59 +00:00
fix(proxy): block every unpriced model a request names
A request can name more than one model, through a comma-separated model or target_model_names on the batch and fine-tuning routes, and the gate only looked at the string case, so an unpriced model riding alongside a priced one went through and billed. Check every candidate and name the unpriced ones in the 403 Aliases had the same problem on the other side: a group that prices itself through its model_info block lands in the cost map under its deployment id, and the explicit-cost check walked the raw model list by group name, so an alias pointing at that group read as unpriced. Resolve the group through the router the way the pricing check already does Also correct the 403 copy. Providers that return their own usage cost still bill for these models, so the accurate claim is that litellm has no pricing of its own for them
This commit is contained in:
parent
df00c334d1
commit
c73480c653
2 changed files with 98 additions and 12 deletions
|
|
@ -513,6 +513,24 @@ def _model_group_has_pricing(model: str, llm_router: "Router") -> bool:
|
|||
return False
|
||||
|
||||
|
||||
def _group_declares_explicit_cost(model: str, llm_router: "Router") -> bool:
|
||||
"""
|
||||
Alias-aware counterpart to ``_is_cost_explicitly_configured``, which resolves the model group
|
||||
the same way ``_model_group_has_pricing`` does. A deployment that prices itself through its
|
||||
``model_info`` block lands in the cost map under its deployment id rather than in its
|
||||
litellm_params, and reaching that entry through the router's own resolution keeps an alias
|
||||
pointing at such a group from being read as unpriced.
|
||||
"""
|
||||
for deployment in llm_router.get_model_list(model_name=model) or ():
|
||||
model_id = (deployment.get("model_info") or _EMPTY_COST_ENTRY).get("id")
|
||||
if model_id is None:
|
||||
continue
|
||||
raw_entry = litellm.model_cost.get(model_id, _EMPTY_COST_ENTRY)
|
||||
if "input_cost_per_token" in raw_entry or "output_cost_per_token" in raw_entry:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def model_has_no_cost_mapping(model: str | None, llm_router: Router | None) -> bool:
|
||||
if not model or llm_router is None:
|
||||
return False
|
||||
|
|
@ -523,7 +541,24 @@ def model_has_no_cost_mapping(model: str | None, llm_router: Router | None) -> b
|
|||
if _model_group_has_pricing(model=model, llm_router=llm_router):
|
||||
return False
|
||||
|
||||
return not _is_cost_explicitly_configured(model, llm_router)
|
||||
return not _group_declares_explicit_cost(model=model, llm_router=llm_router)
|
||||
|
||||
|
||||
def _unpriced_models_in_request(model: str | list[str] | None, llm_router: Router | None) -> tuple[str, ...]:
|
||||
candidates: Final = (model,) if isinstance(model, str) else tuple(model or ())
|
||||
return tuple(
|
||||
candidate for candidate in candidates if model_has_no_cost_mapping(model=candidate, llm_router=llm_router)
|
||||
)
|
||||
|
||||
|
||||
def _unpriced_models_block_message(models: tuple[str, ...]) -> str:
|
||||
names: Final = ", ".join(f"'{model}'" for model in models)
|
||||
subject: Final = f"Model {names} has" if len(models) == 1 else f"Models {names} have"
|
||||
return (
|
||||
f"{subject} no pricing in the cost map, so litellm cannot price the request. "
|
||||
"Requests for unpriced models are blocked because 'block_requests_for_models_without_pricing' "
|
||||
"is enabled. Add pricing (input_cost_per_token/output_cost_per_token) to allow the request."
|
||||
)
|
||||
|
||||
|
||||
async def _run_project_checks(
|
||||
|
|
@ -796,18 +831,14 @@ async def common_checks(
|
|||
and (route in MODEL_DISCOVERY_ROUTES or not RouteChecks.is_llm_api_route(route=route))
|
||||
)
|
||||
|
||||
if (
|
||||
litellm.block_requests_for_models_without_pricing
|
||||
and isinstance(_model, str)
|
||||
and RouteChecks.is_llm_api_route(route=route)
|
||||
and model_has_no_cost_mapping(model=_model, llm_router=llm_router)
|
||||
):
|
||||
unpriced_models: Final = (
|
||||
_unpriced_models_in_request(model=_model, llm_router=llm_router)
|
||||
if litellm.block_requests_for_models_without_pricing and RouteChecks.is_llm_api_route(route=route)
|
||||
else ()
|
||||
)
|
||||
if unpriced_models:
|
||||
raise ProxyException(
|
||||
message=(
|
||||
f"Model '{_model}' has no pricing in the cost map, so its spend would be tracked as $0. "
|
||||
"Requests for unpriced models are blocked because 'block_requests_for_models_without_pricing' "
|
||||
"is enabled. Add pricing for this model (input_cost_per_token/output_cost_per_token) to allow it."
|
||||
),
|
||||
message=_unpriced_models_block_message(unpriced_models),
|
||||
type=ProxyErrorTypes.model_cost_map_missing,
|
||||
param="model",
|
||||
code=status.HTTP_403_FORBIDDEN,
|
||||
|
|
|
|||
|
|
@ -6784,3 +6784,58 @@ async def test_common_checks_blocks_alias_resolving_to_unpriced_model(monkeypatc
|
|||
assert exc_info.value.code == "403"
|
||||
assert exc_info.value.type == ProxyErrorTypes.model_cost_map_missing
|
||||
assert "public-alias" in exc_info.value.message
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_common_checks_blocks_comma_separated_request_carrying_an_unpriced_model(monkeypatch):
|
||||
monkeypatch.setattr(litellm, "block_requests_for_models_without_pricing", True)
|
||||
router = _router_with_priced_and_unpriced_models()
|
||||
|
||||
with pytest.raises(ProxyException) as exc_info:
|
||||
await _run_common_checks(model="priced-group,unpriced-group", llm_router=router)
|
||||
|
||||
assert exc_info.value.code == "403"
|
||||
assert exc_info.value.type == ProxyErrorTypes.model_cost_map_missing
|
||||
assert "'unpriced-group'" in exc_info.value.message
|
||||
assert "'priced-group'" not in exc_info.value.message
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_common_checks_allows_comma_separated_request_when_every_model_is_priced(monkeypatch):
|
||||
monkeypatch.setattr(litellm, "block_requests_for_models_without_pricing", True)
|
||||
router = _router_with_priced_and_unpriced_models()
|
||||
|
||||
result = await _run_common_checks(model="priced-group,priced-group", llm_router=router)
|
||||
|
||||
assert result is True
|
||||
|
||||
|
||||
def _router_with_a_group_priced_through_model_info() -> "Router":
|
||||
from litellm.router import Router
|
||||
|
||||
return Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "model-info-priced-group",
|
||||
"litellm_params": {"model": f"{UNPRICED_UNDERLYING_MODEL}-model-info", "api_key": "sk-test"},
|
||||
"model_info": {"input_cost_per_token": 0, "output_cost_per_token": 0},
|
||||
}
|
||||
],
|
||||
model_group_alias={"model-info-priced-alias": "model-info-priced-group"},
|
||||
)
|
||||
|
||||
|
||||
def test_model_has_no_cost_mapping_group_priced_through_model_info_is_false():
|
||||
from litellm.proxy.auth.auth_checks import model_has_no_cost_mapping
|
||||
|
||||
router = _router_with_a_group_priced_through_model_info()
|
||||
|
||||
assert model_has_no_cost_mapping(model="model-info-priced-group", llm_router=router) is False
|
||||
|
||||
|
||||
def test_model_has_no_cost_mapping_alias_to_a_group_priced_through_model_info_is_false():
|
||||
from litellm.proxy.auth.auth_checks import model_has_no_cost_mapping
|
||||
|
||||
router = _router_with_a_group_priced_through_model_info()
|
||||
|
||||
assert model_has_no_cost_mapping(model="model-info-priced-alias", llm_router=router) is False
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue