diff --git a/litellm/__init__.py b/litellm/__init__.py index 471b273f00d..c8df4394a06 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1402,6 +1402,7 @@ from .exceptions import ( JSONSchemaValidationError, LITELLM_EXCEPTION_TYPES, MockException, + ModelNotMappedError as ModelNotMappedError, ) from .budget_manager import BudgetManager from .proxy.proxy_cli import run_server diff --git a/litellm/exceptions.py b/litellm/exceptions.py index c8de2ab12ed..3bae8a95ef6 100644 --- a/litellm/exceptions.py +++ b/litellm/exceptions.py @@ -991,6 +991,10 @@ LITELLM_EXCEPTION_TYPES: Final = [ ] +class ModelNotMappedError(Exception): + pass + + class BudgetExceededError(Exception): def __init__( self, diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index b5980f9b224..822b827f985 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -2323,6 +2323,13 @@ async def calculate_spend(request: SpendCalculateRequest): param=getattr(e, "param", "None"), code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), ) + if isinstance(e, litellm.exceptions.ModelNotMappedError): + raise ProxyException( + message=str(e), + type="invalid_request_error", + param="model", + code=status.HTTP_400_BAD_REQUEST, + ) error_msg: Final = f"{e}" raise ProxyException( message=getattr(e, "message", error_msg), diff --git a/litellm/utils.py b/litellm/utils.py index b5d396030f7..01f6fee3594 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -472,6 +472,7 @@ from .exceptions import ( BudgetExceededError, ContentPolicyViolationError, ContextWindowExceededError, + ModelNotMappedError, NotFoundError, OpenAIError, PermissionDeniedError, @@ -5830,6 +5831,13 @@ def _is_potential_model_name_in_model_cost( _ABOVE_THRESHOLD_COST_KEY: Final = ABOVE_THRESHOLD_COST_KEY_PATTERN +def _model_not_mapped_message(model: str, custom_llm_provider: str | None) -> str: + return ( + f"This model isn't mapped yet. model={model}, custom_llm_provider={custom_llm_provider}. " + "Add it here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json." + ) + + def _get_model_info_helper( model: str, custom_llm_provider: str | None = None, @@ -6012,9 +6020,7 @@ def _get_model_info_helper( key, _model_info = generalization if _model_info is None or key is None: - raise ValueError( - "This model isn't mapped yet. Add it here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json" - ) + raise ModelNotMappedError(_model_not_mapped_message(model, custom_llm_provider)) _input_cost_per_token: float | None = _model_info.get("input_cost_per_token") if _input_cost_per_token is None: # default value to 0, be noisy about this @@ -6249,11 +6255,11 @@ def _get_model_info_helper( if cost_key not in returned_model_info and _ABOVE_THRESHOLD_COST_KEY.search(cost_key) is not None: returned_model_info[cost_key] = cost_value return returned_model_info + except ModelNotMappedError: + raise except Exception as e: verbose_logger.debug("Error getting model info: %s", e) - raise Exception( - f"This model isn't mapped yet. model={model}, custom_llm_provider={custom_llm_provider}. Add it here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json." - ) + raise Exception(_model_not_mapped_message(model, custom_llm_provider)) def _build_model_info( diff --git a/tests/integration/contracts.json b/tests/integration/contracts.json index cdf534bb5a1..1ae760467f1 100644 --- a/tests/integration/contracts.json +++ b/tests/integration/contracts.json @@ -275,6 +275,9 @@ "tests/integration/spend/test_filtered_ledger.py::test_rotated_keys_users_and_model_groups_preserve_success_failure_cache_ledger": [ "quota_management.spend_tracking.filtered_ledger_preserves_owner_identity_and_totals" ], + "tests/integration/spend/test_spend_calculate.py::test_spend_calculate_rejects_unpriced_model_with_400": [ + "quota_management.spend_tracking.spend_calculate.rejects_unpriced_model" + ], "tests/integration/management/test_partial_update_sequences.py::test_restricted_actor_cannot_detach_key_from_project": [ "mgmt.key.update.project_detach_denied_to_restricted_actor" ], diff --git a/tests/integration/spend/test_spend_calculate.py b/tests/integration/spend/test_spend_calculate.py new file mode 100644 index 00000000000..855dd2b21f3 --- /dev/null +++ b/tests/integration/spend/test_spend_calculate.py @@ -0,0 +1,20 @@ +import uuid +from typing import Final + +import pytest +from integration._support.client import JSON_OBJECT, Gateway, object_value, string_value + + +@pytest.mark.covers("quota_management.spend_tracking.spend_calculate.rejects_unpriced_model") +def test_spend_calculate_rejects_unpriced_model_with_400(gateway: Gateway) -> None: + model: Final = f"openrouter/integration-unpriced-{uuid.uuid4().hex}" + response: Final = gateway.request( + "POST", + "/spend/calculate", + {"model": model, "messages": [{"role": "user", "content": "price this request"}]}, + ) + assert response.status_code == 400, response.text + error: Final = object_value(JSON_OBJECT.validate_json(response.text)["error"]) + assert error["type"] == "invalid_request_error", response.text + assert error["param"] == "model", response.text + assert model in string_value(error["message"]), response.text diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py index e41c027d962..c6a4173b583 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py @@ -257,6 +257,8 @@ from litellm.constants import LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME from litellm.proxy._types import ( LitellmUserRoles, Member, + ProxyException, + SpendCalculateRequest, SpendLogsPayload, UserAPIKeyAuth, ) @@ -7835,3 +7837,18 @@ def test_ui_view_request_response_internal_user_missing_row_forbidden(client, mo assert custom_logger.requested_ids == [] finally: app.dependency_overrides.pop(ps.user_api_key_auth, None) + + +@pytest.mark.asyncio +async def test_calculate_spend_unpriced_model_returns_400(): + model = "openrouter/unit-test-unpriced-model" + with patch("litellm.proxy.proxy_server.llm_router", None): + with pytest.raises(ProxyException) as exc_info: + await spend_management_endpoints.calculate_spend( + SpendCalculateRequest(model=model, messages=[{"role": "user", "content": "hi"}]) + ) + + assert exc_info.value.code == "400" + assert exc_info.value.type == "invalid_request_error" + assert exc_info.value.param == "model" + assert model in exc_info.value.message diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 07982f51153..79462d16a8c 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -230,6 +230,13 @@ def test_get_model_info_prefers_exact_dated_key_over_stripped( assert info["key"] == expected_key +def test_get_model_info_internal_failure_is_not_reported_as_unmapped() -> None: + with patch("litellm.utils._get_potential_model_names", side_effect=RuntimeError("malformed metadata")): + with pytest.raises(Exception, match="This model isn't mapped yet") as exc_info: + litellm.utils._get_model_info_helper(model="gpt-4o", custom_llm_provider="openai") + assert not isinstance(exc_info.value, litellm.ModelNotMappedError) + + def test_check_provider_match_azure_ai_allows_openai_and_azure(): """ Test that azure_ai provider can match openai and azure models.