diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 404fe400a16..3b34440d0bf 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -403,6 +403,7 @@ class LiteLLMRoutes(enum.Enum): "/v1/models", # token counter "/utils/token_counter", + "/utils/model_info", "/utils/transform_request", # rerank "/rerank", diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 2ae285d6eef..9d1a4065f31 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -845,6 +845,7 @@ MODEL_DISCOVERY_ROUTES: Final = frozenset( "/v1/model/info", "/v2/model/info", "/model_group/info", + "/utils/model_info", } ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 7634237a59f..3a06753834a 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -13476,6 +13476,48 @@ async def supported_openai_params(model: str): raise HTTPException(status_code=400, detail={"error": f"Could not map model={model}"}) +class _ModelInfoLookupResponse(TypedDict): + model: ReadOnly[str] + custom_llm_provider: ReadOnly[str] + model_info: ReadOnly[Mapping[str, object]] + + +@router.get( + "/utils/model_info", + tags=["llm utils"], # mutable-ok: FastAPI tags kwarg is list-typed + dependencies=[Depends(user_api_key_auth)], # mutable-ok: FastAPI dependencies kwarg is list-typed +) +async def model_info_lookup(model: str, custom_llm_provider: str | None = None): + """ + Returns the model cost map entry (token limits, pricing, supports_* capabilities) for any model + in the cost map, whether or not it is registered on this proxy. `model_info` carries every + field of the raw cost map entry plus the typed fields `litellm.get_model_info` derives from it + (`key`, `supported_openai_params`). + + Example curl: + ``` + curl -X GET --location 'http://localhost:4000/utils/model_info?model=gpt-4o&custom_llm_provider=openai' \ + --header 'Authorization: Bearer sk-1234' + ``` + """ + detail: Final = { # mutable-ok: FastAPI serializes detail as a plain dict + "error": f"model={model}, custom_llm_provider={custom_llm_provider} is not in the model cost map" + } + try: + typed_model_info: Final = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider) + except Exception: + raise HTTPException(status_code=404, detail=detail) + cost_map_entry: Final = litellm.model_cost.get(typed_model_info["key"]) + if cost_map_entry is None: + raise HTTPException(status_code=404, detail=detail) + response: Final[_ModelInfoLookupResponse] = { + "model": model, + "custom_llm_provider": typed_model_info["litellm_provider"], + "model_info": {**typed_model_info, **cost_map_entry}, + } + return response + + @router.post( "/utils/transform_request", tags=["llm utils"], diff --git a/tests/test_litellm/proxy/auth/test_auth_checks.py b/tests/test_litellm/proxy/auth/test_auth_checks.py index a0256e40b8c..2824708d502 100644 --- a/tests/test_litellm/proxy/auth/test_auth_checks.py +++ b/tests/test_litellm/proxy/auth/test_auth_checks.py @@ -67,6 +67,7 @@ from litellm.constants import ( REGISTRY_ERROR_NEGATIVE_CACHE_TTL, TAG_REGISTRY_MAX_SIZE, ) +from litellm.proxy.auth.route_checks import RouteChecks from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper from litellm.proxy.common_utils.user_api_key_cache import ( END_USER_RESTRICTED_REGISTRY_OVERFLOW_SENTINEL, @@ -8889,6 +8890,8 @@ def test_jwt_team_role_reaches_the_gateway_token_endpoint_by_default(): def test_route_skips_budget_checks_marks_only_spend_free_routes() -> None: assert route_skips_budget_checks(route="/v1/models") is True assert route_skips_budget_checks(route="/spend/logs") is True + assert route_skips_budget_checks(route="/utils/model_info") is True + assert RouteChecks.is_llm_api_route(route="/utils/model_info") is True assert route_skips_budget_checks(route="/health") is False assert route_skips_budget_checks(route="/v1/chat/completions") is False diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_utils.py b/tests/test_litellm/proxy/proxy_server/test_routes_utils.py index 1e1436fcef8..b363d3823ad 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_utils.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_utils.py @@ -3,6 +3,7 @@ Pins (PR2): - POST /utils/token_counter - GET /utils/supported_openai_params + - GET /utils/model_info - POST /utils/transform_request """ @@ -231,6 +232,66 @@ def test_supported_openai_params_invalid_model(client, auth_as, monkeypatch): assert "Could not map model" in response.text +# --------------------------------------------------------------------------- +# GET /utils/model_info +# --------------------------------------------------------------------------- + + +@pytest.fixture +def lookup_fixture_model(monkeypatch): + entry = { + "litellm_provider": "openai", + "mode": "chat", + "max_input_tokens": 1234, + "max_output_tokens": 56, + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "supports_vision": True, + "deprecation_date": "2099-01-01", + "supports_lookup_fixture_edit": True, + } + monkeypatch.setattr(proxy_server, "llm_router", None) + monkeypatch.setitem(litellm.model_cost, "lookup-fixture-model", entry) + litellm.get_model_info.cache_clear() + litellm.utils._cached_get_model_info_helper.cache_clear() + yield entry + litellm.get_model_info.cache_clear() + litellm.utils._cached_get_model_info_helper.cache_clear() + + +def test_model_info_lookup_returns_full_cost_map_entry_for_unregistered_model(client, auth_as, lookup_fixture_model): + """Every raw cost map field comes back, including ones outside ``ModelInfoBase`` that ``get_model_info`` drops.""" + with auth_as(): + response = client.get( + "/utils/model_info", params={"model": "lookup-fixture-model", "custom_llm_provider": "openai"} + ) + assert response.status_code == 200, response.text + body = response.json() + assert body["model"] == "lookup-fixture-model" + assert body["custom_llm_provider"] == "openai" + assert body["model_info"]["key"] == "lookup-fixture-model" + assert isinstance(body["model_info"]["supported_openai_params"], list) + assert {k: body["model_info"][k] for k in lookup_fixture_model} == lookup_fixture_model + + +def test_model_info_lookup_unknown_model_returns_404(client, auth_as, monkeypatch): + monkeypatch.setattr(proxy_server, "llm_router", None) + with auth_as(): + response = client.get("/utils/model_info", params={"model": "no-such-model-lit-7476"}) + assert response.status_code == 404, response.text + assert "is not in the model cost map" in response.text + + +def test_model_info_lookup_returns_404_when_typed_info_has_no_cost_map_entry(client, auth_as, monkeypatch): + """``get_model_info`` synthesizes info for huggingface fallbacks absent from ``model_cost``; + with no raw entry the route must 404 rather than answer 200 with typed fields only.""" + monkeypatch.setattr(proxy_server, "llm_router", None) + with auth_as(): + response = client.get("/utils/model_info", params={"model": "huggingface/not-in-map-org/not-in-map-model"}) + assert response.status_code == 404, response.text + assert "is not in the model cost map" in response.text + + # --------------------------------------------------------------------------- # POST /utils/transform_request # --------------------------------------------------------------------------- diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index c4b2a92f79e..29b7cd206eb 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -17448,6 +17448,34 @@ export interface paths { patch?: never; trace?: never; }; + "/utils/model_info": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Model Info Lookup + * @description Returns the model cost map entry (token limits, pricing, supports_* capabilities) for any model + * in the cost map, whether or not it is registered on this proxy. `model_info` carries every + * field of the raw cost map entry plus the typed fields `litellm.get_model_info` derives from it + * (`key`, `supported_openai_params`). + * + * Example curl: + * ``` + * curl -X GET --location 'http://localhost:4000/utils/model_info?model=gpt-4o&custom_llm_provider=openai' --header 'Authorization: Bearer sk-1234' + * ``` + */ + get: operations["model_info_lookup_utils_model_info_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/utils/supported_openai_params": { parameters: { query?: never; @@ -63647,6 +63675,38 @@ export interface operations { }; }; }; + model_info_lookup_utils_model_info_get: { + parameters: { + query: { + model: string; + custom_llm_provider?: string | null; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; supported_openai_params_utils_supported_openai_params_get: { parameters: { query: {