diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index b4b23f111d1..50792d03976 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -933,6 +933,8 @@ from litellm.types.llms.openai import ( from litellm.types.proxy.control_plane_endpoints import WorkerRegistryEntry from litellm.types.proxy.management_endpoints.model_management_endpoints import ( ModelGroupInfoProxy, + ModelGroupInfoResponse, + ModelInfoV1Response, ) from litellm.types.proxy.management_endpoints.ui_sso import ( DefaultTeamSSOParams, @@ -16371,11 +16373,13 @@ def _model_info_json_response(data: Sequence[Mapping[str, object]] | Mapping[str "/model/info", tags=["model management"], dependencies=[Depends(user_api_key_auth)], + response_model=ModelInfoV1Response, ) @router.get( "/v1/model/info", tags=["model management"], dependencies=[Depends(user_api_key_auth)], + response_model=ModelInfoV1Response, ) async def model_info_v1( user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), @@ -16677,11 +16681,12 @@ _get_model_group_info: Final = get_model_group_info "/model_group/info", tags=["model management"], dependencies=[Depends(user_api_key_auth)], + response_model=ModelGroupInfoResponse, ) async def model_group_info( user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), model_group: str | None = None, -): +) -> ModelGroupInfoResponse: """ Get information about all the deployments on litellm proxy, including config.yaml descriptions (except api key and api base) @@ -16836,7 +16841,7 @@ async def model_group_info( # Return empty data array when no models are configured (graceful handling for fresh installs) if llm_model_list is None or llm_router is None or not llm_model_list: - return {"data": []} + return ModelGroupInfoResponse(data=()) from litellm.proxy.utils import get_available_models_for_user @@ -16894,13 +16899,13 @@ async def model_group_info( user_api_key_dict, tuple(dict.fromkeys(public_group_names)) ) - return { - "data": [ + return ModelGroupInfoResponse( + data=tuple( group for group, public_name in zip(model_groups, public_group_names, strict=True) if public_name not in callback_hidden_names - ] - } + ) + ) @router.get( diff --git a/litellm/types/proxy/management_endpoints/model_management_endpoints.py b/litellm/types/proxy/management_endpoints/model_management_endpoints.py index 3cc85a9427e..4ab6be4cefa 100644 --- a/litellm/types/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/types/proxy/management_endpoints/model_management_endpoints.py @@ -5,7 +5,8 @@ from pydantic import ConfigDict, Field from litellm.types.llms.base import LiteLLMBaseModel -from ...router import ModelGroupInfo +from ...router import ModelGroupInfo, ModelInfo +from .management_v1 import ResourceResponse class ModelGroupInfoProxy(ModelGroupInfo): @@ -15,6 +16,61 @@ class ModelGroupInfoProxy(ModelGroupInfo): health_checked_at: str | None = Field(default=None) +class ModelGroupInfoResponse(ResourceResponse[tuple[ModelGroupInfoProxy, ...]]): + """`{data: [...]}` of `GET /model_group/info`: one `ModelGroupInfoProxy` per model group the caller may see.""" + + model_config = ConfigDict(frozen=True) + + +class ModelInfoV1LiteLLMParams(LiteLLMBaseModel): + """A deployment's litellm_params as `GET /model/info` serves them. + + Every configured key passes through except credentials: api_key, client_secret, vertex credentials and AWS + access keys are removed and every other secret-shaped value is masked, except litellm_credential_name. With the + proxy started from the CLI with `--model`, every unset parameter arrives as the string "None", or as an empty + string when its name is secret-shaped. + """ + + model_config = ConfigDict(frozen=True, extra="allow") + + model: str + + +class ModelInfoV1Deployment(LiteLLMBaseModel): + """One deployment as `GET /model/info` and `GET /v1/model/info` report it. + + Any further deployment-level key passes through from the router row unchanged. + """ + + model_config = ConfigDict(frozen=True, extra="allow") + + model_name: str = Field( + description=( + "The name requests use for this deployment. A team deployment stored under the proxy's internal " + "`model_name__` name reports its team_public_model_name instead." + ) + ) + litellm_params: ModelInfoV1LiteLLMParams + model_info: ModelInfo = Field( + description=( + "The configured model_info merged with the cost map entry for the model (pricing, token limits, " + "mode, supports_* capabilities, supported_openai_params, litellm_provider, key), the proxy's " + "discovered model info and pricing_overrides. With the proxy database connected, direct_access is " + "added, and access_via_team_ids for an admin or for a key that belongs to a user." + ) + ) + + +class ModelInfoV1Response(ResourceResponse[tuple[ModelInfoV1Deployment, ...] | ModelInfoV1Deployment]): + """`{data: ...}` of `GET /model/info` and `GET /v1/model/info`. + + `data` is one `ModelInfoV1Deployment` per deployment the caller may see. With the proxy started from the CLI + with `--model` and no config, `data` is that single deployment as one object instead of a list. + """ + + model_config = ConfigDict(frozen=True) + + class UpdateUsefulLinksRequest(LiteLLMBaseModel): # Supports both old format (Dict[str, str]) and new format (Dict[str, Dict[str, Any]]) # New format: { "displayName": { "url": "...", "index": 0 } } diff --git a/tests/unit/proxy/proxy_server/test_routes_model_info.py b/tests/unit/proxy/proxy_server/test_routes_model_info.py index fae4a6470cf..d88ed10375f 100644 --- a/tests/unit/proxy/proxy_server/test_routes_model_info.py +++ b/tests/unit/proxy/proxy_server/test_routes_model_info.py @@ -10,7 +10,7 @@ Pins (PR2): from __future__ import annotations import copy -from collections.abc import Callable +from collections.abc import Callable, Mapping from contextlib import AbstractContextManager from typing import Final from unittest.mock import AsyncMock, MagicMock @@ -214,6 +214,42 @@ def test_v2_model_info_in_openapi_schema(): assert "get" in schema["paths"]["/v2/model/info"] +_MODEL_INFO_V1_ITEM: Final = {"$ref": "#/components/schemas/ModelInfoV1Deployment"} + + +@pytest.mark.parametrize( + ("path", "response_schema", "data_schema"), + ( + ( + "/model/info", + "ModelInfoV1Response", + {"anyOf": [{"items": _MODEL_INFO_V1_ITEM, "type": "array"}, _MODEL_INFO_V1_ITEM]}, + ), + ( + "/v1/model/info", + "ModelInfoV1Response", + {"anyOf": [{"items": _MODEL_INFO_V1_ITEM, "type": "array"}, _MODEL_INFO_V1_ITEM]}, + ), + ( + "/model_group/info", + "ModelGroupInfoResponse", + {"items": {"$ref": "#/components/schemas/ModelGroupInfoProxy"}, "type": "array"}, + ), + ), +) +def test_model_info_routes_declare_their_response_schema( + path: str, response_schema: str, data_schema: Mapping[str, object] +) -> None: + """The OpenAPI document types the 200 body of each model-info route, so a client can generate types.""" + from litellm.proxy.proxy_server import get_openapi_schema + + schema: Final = get_openapi_schema() + declared: Final = schema["paths"][path]["get"]["responses"]["200"]["content"]["application/json"]["schema"] + assert declared == {"$ref": f"#/components/schemas/{response_schema}"} + data: Final = schema["components"]["schemas"][response_schema]["properties"]["data"] + assert {key: value for key, value in data.items() if key != "title"} == data_schema + + # --------------------------------------------------------------------------- # GET /v1/model/info, GET /model/info # --------------------------------------------------------------------------- @@ -264,6 +300,88 @@ def test_v1_model_info_specific_id_happy(client, auth_as, configured_router, pat } +@pytest.fixture +def real_router(monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None) -> litellm.Router: + """A real Router with a cost-map model, a wildcard and a model the cost map does not know.""" + router: Final = litellm.Router( + model_list=[ + { + "model_name": "gpt-4o", + "litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-test"}, + "model_info": {"id": "known-id"}, + }, + { + "model_name": "openai/*", + "litellm_params": {"model": "openai/*", "api_key": "sk-test"}, + "model_info": {"id": "wildcard-id"}, + }, + { + "model_name": "mystery", + "litellm_params": { + "model": "openai/mystery-model", + "api_key": "sk-test", + "api_base": "https://mystery.example/v1", + }, + "model_info": {"id": "unknown-id"}, + }, + ] + ) + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "llm_model_list", router.get_model_list()) + monkeypatch.setattr(proxy_server, "user_model", None) + monkeypatch.setattr(proxy_server, "prisma_client", None) + return router + + +@pytest.mark.parametrize("path", ["/v1/model/info", "/model/info"]) +def test_v1_model_info_bodies_validate_against_the_declared_response_model( + client: TestClient, + auth_as: Callable[[], AbstractContextManager[object]], + real_router: litellm.Router, + path: str, +) -> None: + """The orjson body, which FastAPI never validates, fits the declared schema for a cost-map model, a wildcard + expansion and an unknown model, both listed and looked up by id, with credentials gone.""" + from litellm.types.proxy.management_endpoints.model_management_endpoints import ModelInfoV1Response + + with auth_as(): + listing: Final = client.get(path) + by_id: Final = client.get(path, params={"litellm_model_id": "unknown-id"}) + assert (listing.status_code, by_id.status_code) == (200, 200), (listing.text, by_id.text) + listed: Final = ModelInfoV1Response.model_validate(listing.json()).data + assert isinstance(listed, tuple) + assert {deployment.model_name for deployment in listed} > {"gpt-4o", "mystery"} + assert all("api_key" not in deployment.litellm_params.model_dump() for deployment in listed) + looked_up: Final = ModelInfoV1Response.model_validate(by_id.json()).data + assert isinstance(looked_up, tuple) + assert [deployment.model_name for deployment in looked_up] == ["mystery"] + + +@pytest.mark.parametrize("path", ["/v1/model/info", "/model/info"]) +def test_v1_model_info_cli_model_body_is_one_deployment( + client: TestClient, + auth_as: Callable[[], AbstractContextManager[object]], + real_router: litellm.Router, + monkeypatch: pytest.MonkeyPatch, + path: str, +) -> None: + """Started as ``litellm --model``, the route serves ``data`` as one deployment object, the second shape the + schema declares, with an unset non-secret parameter arriving as the string "None".""" + from litellm.types.proxy.management_endpoints.model_management_endpoints import ( + ModelInfoV1Deployment, + ModelInfoV1Response, + ) + + monkeypatch.setattr(proxy_server, "user_model", "gpt-4o") + with auth_as(): + response: Final = client.get(path) + assert response.status_code == 200, response.text + declared: Final = ModelInfoV1Response.model_validate(response.json()).data + assert isinstance(declared, ModelInfoV1Deployment) + assert (declared.model_name, declared.litellm_params.model) == ("*", "gpt-4o") + assert declared.litellm_params.model_dump()["max_retries"] == "None" + + @pytest.mark.parametrize("path", ["/v1/model/info", "/model/info"]) def test_v1_model_info_no_model_list_error(client, auth_as, null_router, path): """Pins ``GET /v1/model/info`` and ``GET /model/info`` (error: no model list).""" diff --git a/tests/unit/proxy/test_model_list_callback_filter.py b/tests/unit/proxy/test_model_list_callback_filter.py index d397ee676f2..ae9d591d65a 100644 --- a/tests/unit/proxy/test_model_list_callback_filter.py +++ b/tests/unit/proxy/test_model_list_callback_filter.py @@ -153,7 +153,7 @@ async def _v1_model_info_names(user_api_key_dict: UserAPIKeyAuth) -> list[str]: async def _model_groups(user_api_key_dict: UserAPIKeyAuth) -> list[str]: response = await proxy_server.model_group_info(user_api_key_dict=user_api_key_dict) - return [group.model_group for group in response["data"]] + return [group.model_group for group in response.data] async def _model_by_id_status(model_id: str, user_api_key_dict: UserAPIKeyAuth) -> int: diff --git a/tests/unit/proxy/test_model_list_discoverable.py b/tests/unit/proxy/test_model_list_discoverable.py index 0be5342c45c..cd93a50afcb 100644 --- a/tests/unit/proxy/test_model_list_discoverable.py +++ b/tests/unit/proxy/test_model_list_discoverable.py @@ -112,7 +112,7 @@ async def _v1_model_info_names(user_api_key_dict: UserAPIKeyAuth, **kwargs) -> l async def _model_groups(user_api_key_dict: UserAPIKeyAuth) -> list[str]: response = await proxy_server.model_group_info(user_api_key_dict=user_api_key_dict) - return [group.model_group for group in response["data"]] + return [group.model_group for group in response.data] @pytest.mark.asyncio diff --git a/tests/unit/proxy/test_proxy_server.py b/tests/unit/proxy/test_proxy_server.py index 79f4edd4d62..e77c370bf98 100644 --- a/tests/unit/proxy/test_proxy_server.py +++ b/tests/unit/proxy/test_proxy_server.py @@ -2369,7 +2369,7 @@ async def test_proxy_model_group_alias_checks(prisma_client, hidden): # noqa: F user_api_key_dict=UserAPIKeyAuth(models=[]), ) print(f"resp: {resp}") - models = resp["data"] + models = resp.data is_model_alias_in_list = False print(f"model_alias: {model_alias}, models: {models}") for item in models: @@ -2438,7 +2438,7 @@ async def test_proxy_model_group_info_rerank(prisma_client): # noqa: F811 # py ) print(resp) - models = resp["data"] + models = resp.data assert models[0].mode == "rerank" diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 1108d4a8743..af8962f36b9 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -39054,11 +39054,132 @@ export interface components { /** Tpm */ tpm?: number | null; }; + /** + * ModelGroupInfoResponse + * @description `{data: [...]}` of `GET /model_group/info`: one `ModelGroupInfoProxy` per model group the caller may see. + */ + ModelGroupInfoResponse: { + /** Data */ + data: components["schemas"]["ModelGroupInfoProxy"][]; + }; + /** ModelInfo */ + "ModelInfo-Output": { + /** Access Windows */ + access_windows?: components["schemas"]["ModelAccessWindow"][] | null; + /** Allow Fail Open */ + allow_fail_open?: boolean | null; + /** Base Model */ + base_model?: string | null; + /** Blocked */ + blocked?: boolean | null; + /** Cache Creation Input Token Cost */ + cache_creation_input_token_cost?: number | null; + /** Cache Read Input Token Cost */ + cache_read_input_token_cost?: number | null; + /** Cost Per Ptu Per Hour */ + cost_per_ptu_per_hour?: number | null; + /** Created At */ + created_at?: string | null; + /** Created By */ + created_by?: string | null; + /** + * Db Model + * @default false + */ + db_model: boolean; + /** Discoverable */ + discoverable?: boolean | null; + /** Enable Tag Filtering */ + enable_tag_filtering?: boolean | null; + /** Id */ + id: string | null; + /** Input Cost Per Character */ + input_cost_per_character?: number | null; + /** Input Cost Per Token */ + input_cost_per_token?: number | null; + /** Internal Router Model */ + internal_router_model?: boolean | null; + /** + * Member Auto Router + * @default false + */ + member_auto_router: boolean; + /** Output Cost Per Character */ + output_cost_per_character?: number | null; + /** Output Cost Per Token */ + output_cost_per_token?: number | null; + /** Ptu Count */ + ptu_count?: number | null; + /** Ptu Effective From */ + ptu_effective_from?: string | null; + /** Ptu Effective To */ + ptu_effective_to?: string | null; + /** Team Id */ + team_id?: string | null; + /** Team Public Model Name */ + team_public_model_name?: string | null; + /** Tier */ + tier?: ("free" | "paid") | null; + /** Tiered Pricing */ + tiered_pricing?: { + [key: string]: unknown; + }[] | null; + /** Updated At */ + updated_at?: string | null; + /** Updated By */ + updated_by?: string | null; + } & { + [key: string]: unknown; + }; /** ModelInfoDelete */ ModelInfoDelete: { /** Id */ id: string; }; + /** + * ModelInfoV1Deployment + * @description One deployment as `GET /model/info` and `GET /v1/model/info` report it. + * + * Any further deployment-level key passes through from the router row unchanged. + */ + ModelInfoV1Deployment: { + litellm_params: components["schemas"]["ModelInfoV1LiteLLMParams"]; + /** @description The configured model_info merged with the cost map entry for the model (pricing, token limits, mode, supports_* capabilities, supported_openai_params, litellm_provider, key), the proxy's discovered model info and pricing_overrides. With the proxy database connected, direct_access is added, and access_via_team_ids for an admin or for a key that belongs to a user. */ + model_info: components["schemas"]["ModelInfo-Output"]; + /** + * Model Name + * @description The name requests use for this deployment. A team deployment stored under the proxy's internal `model_name__` name reports its team_public_model_name instead. + */ + model_name: string; + } & { + [key: string]: unknown; + }; + /** + * ModelInfoV1LiteLLMParams + * @description A deployment's litellm_params as `GET /model/info` serves them. + * + * Every configured key passes through except credentials: api_key, client_secret, vertex credentials and AWS + * access keys are removed and every other secret-shaped value is masked, except litellm_credential_name. With the + * proxy started from the CLI with `--model`, every unset parameter arrives as the string "None", or as an empty + * string when its name is secret-shaped. + */ + ModelInfoV1LiteLLMParams: { + /** Model */ + model: string; + } & { + [key: string]: unknown; + }; + /** + * ModelInfoV1Response + * @description `{data: ...}` of `GET /model/info` and `GET /v1/model/info`. + * + * `data` is one `ModelInfoV1Deployment` per deployment the caller may see. With the proxy started from the CLI + * with `--model` and no config, `data` is that single deployment as one object instead of a list. + */ + ModelInfoV1Response: { + /** Data */ + data: components["schemas"]["ModelInfoV1Deployment"][] | components["schemas"]["ModelInfoV1Deployment"]; + }; /** ModelInsightDailyMetric */ ModelInsightDailyMetric: { /** Completion Tokens */ @@ -66194,7 +66315,7 @@ export interface operations { [name: string]: unknown; }; content: { - "application/json": unknown; + "application/json": components["schemas"]["ModelInfoV1Response"]; }; }; /** @description Validation Error */ @@ -66520,7 +66641,7 @@ export interface operations { [name: string]: unknown; }; content: { - "application/json": unknown; + "application/json": components["schemas"]["ModelGroupInfoResponse"]; }; }; /** @description Validation Error */ @@ -80838,7 +80959,7 @@ export interface operations { [name: string]: unknown; }; content: { - "application/json": unknown; + "application/json": components["schemas"]["ModelInfoV1Response"]; }; }; /** @description Validation Error */