diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 43d2a149758..e7b8dd01dae 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -15036,6 +15036,13 @@ def _get_proxy_model_info(model: dict) -> dict: return _translate_model_name_for_response(model) +def _model_info_json_response(data: Sequence[Mapping[str, object]] | Mapping[str, object]) -> Response: + return Response( + content=orjson.dumps({"data": data}, default=jsonable_encoder, option=orjson.OPT_NON_STR_KEYS), + media_type="application/json", + ) + + @router.get( "/model/info", tags=["model management"], @@ -15083,7 +15090,7 @@ async def model_info_v1( `model_info.direct_access` when the proxy database is connected. Returns: - Returns a dictionary containing information about each model. + A JSON response whose `data` list holds one entry per model. Example Response: ```json @@ -15131,7 +15138,7 @@ async def model_info_v1( deployment_dict=_deployment_info_dict, excluded_keys={"litellm_credential_name"}, ) - return {"data": _deployment_info_dict} + return _model_info_json_response(_deployment_info_dict) if llm_model_list is None: raise HTTPException( @@ -15182,7 +15189,7 @@ async def model_info_v1( llm_router=llm_router, user_api_key_dict=user_api_key_dict, ) - return {"data": single_model_list} + return _model_info_json_response(single_model_list) # Return router deployments (same source as /v2/model/info), not wildcard- # expanded model names from get_complete_model_list(). Team-scoped rows @@ -15250,7 +15257,7 @@ async def model_info_v1( visible_models: Final = [model for model in all_models if model.get("model_name") not in hidden_names] verbose_proxy_logger.debug("all_models: %s", visible_models) - return {"data": visible_models} + return _model_info_json_response(visible_models) @router.get( diff --git a/tests/proxy_unit_tests/test_proxy_server.py b/tests/proxy_unit_tests/test_proxy_server.py index d06eb0426c9..9c8dd90dd2b 100644 --- a/tests/proxy_unit_tests/test_proxy_server.py +++ b/tests/proxy_unit_tests/test_proxy_server.py @@ -2139,7 +2139,7 @@ async def test_model_info_alias_without_prisma(hidden): user_api_key_dict=UserAPIKeyAuth(models=[]), ) - models = resp["data"] + models = json.loads(resp.body)["data"] alias_found = any( m["model_name"] == model_alias @@ -2203,7 +2203,7 @@ async def test_proxy_model_group_alias_checks(prisma_client, hidden): # noqa: F resp = await model_info_v1( user_api_key_dict=UserAPIKeyAuth(models=[]), ) - models = resp["data"] + models = json.loads(resp.body)["data"] is_model_alias_in_list = False for item in models: if model_alias == item["model_name"]: @@ -2280,7 +2280,7 @@ async def test_proxy_model_group_info_rerank(prisma_client): # noqa: F811 # py resp = await model_info_v1( user_api_key_dict=UserAPIKeyAuth(models=[]), ) - models = resp["data"] + models = json.loads(resp.body)["data"] assert models[0]["model_info"]["mode"] == "rerank" resp = await model_group_info( user_api_key_dict=UserAPIKeyAuth(models=[]), diff --git a/tests/test_litellm/proxy/proxy_server/test_team_model_name_translation.py b/tests/test_litellm/proxy/proxy_server/test_team_model_name_translation.py index bc346874e0d..baa032f75e6 100644 --- a/tests/test_litellm/proxy/proxy_server/test_team_model_name_translation.py +++ b/tests/test_litellm/proxy/proxy_server/test_team_model_name_translation.py @@ -9,6 +9,7 @@ rows instead of the internal routing key `model_name_{team_id}_{uuid}`. from __future__ import annotations +import json from unittest.mock import AsyncMock, MagicMock import pytest @@ -238,7 +239,7 @@ async def test_model_info_v1_list_path_translates_team_model_name(monkeypatch): ) resp = await ps.model_info_v1(user_api_key_dict=admin, litellm_model_id=None) - names = [m["model_name"] for m in resp["data"]] + names = [m["model_name"] for m in json.loads(resp.body)["data"]] assert "team-claude-sonnet" in names assert "model_name_team-abc-123_4a6b8" not in names @@ -271,7 +272,7 @@ async def test_model_info_v1_unrestricted_key_returns_all_deployments(monkeypatc ) resp = await ps.model_info_v1(user_api_key_dict=caller, litellm_model_id=None) - assert [m["model_name"] for m in resp["data"]] == ["gpt-4"] + assert [m["model_name"] for m in json.loads(resp.body)["data"]] == ["gpt-4"] @pytest.mark.asyncio @@ -303,7 +304,7 @@ async def test_model_info_v1_restricted_key_filters_deployments(monkeypatch): ) resp = await ps.model_info_v1(user_api_key_dict=caller, litellm_model_id=None) - assert [m["model_name"] for m in resp["data"]] == ["gpt-4"] + assert [m["model_name"] for m in json.loads(resp.body)["data"]] == ["gpt-4"] def _other_team_row() -> dict: @@ -367,10 +368,11 @@ async def test_model_info_v1_unrestricted_key_hides_other_team_byok(monkeypatch) ) resp = await ps.model_info_v1(user_api_key_dict=caller, litellm_model_id=None) - returned_ids = {m["model_info"]["id"] for m in resp["data"]} + data = json.loads(resp.body)["data"] + returned_ids = {m["model_info"]["id"] for m in data} assert returned_ids == {"global-id-1", "byok-id-1"} assert "byok-id-other" not in returned_ids - names = [m["model_name"] for m in resp["data"]] + names = [m["model_name"] for m in data] assert "team-claude-sonnet" in names assert "gpt-4" in names @@ -412,7 +414,7 @@ async def test_model_info_v1_service_key_hides_all_team_byok(monkeypatch): ) resp = await ps.model_info_v1(user_api_key_dict=caller, litellm_model_id=None) - assert [m["model_info"]["id"] for m in resp["data"]] == ["global-id-1"] + assert [m["model_info"]["id"] for m in json.loads(resp.body)["data"]] == ["global-id-1"] @pytest.mark.asyncio @@ -466,7 +468,7 @@ async def test_model_info_v1_team_key_sees_own_byok_regardless_of_user_lookup( ) resp = await ps.model_info_v1(user_api_key_dict=caller, litellm_model_id=None) - assert [m["model_info"]["id"] for m in resp["data"]] == ["byok-id-1", "global-id-1"] + assert [m["model_info"]["id"] for m in json.loads(resp.body)["data"]] == ["byok-id-1", "global-id-1"] @pytest.mark.asyncio @@ -509,7 +511,7 @@ async def test_model_info_v1_user_team_membership_grants_byok(monkeypatch): ) resp = await ps.model_info_v1(user_api_key_dict=caller, litellm_model_id=None) - assert [m["model_info"]["id"] for m in resp["data"]] == [ + assert [m["model_info"]["id"] for m in json.loads(resp.body)["data"]] == [ "byok-id-other", "global-id-1", ] @@ -557,7 +559,7 @@ async def test_model_info_v1_populates_access_via_team_ids(monkeypatch): ) resp = await ps.model_info_v1(user_api_key_dict=admin, litellm_model_id=None) - by_id = {m["model_info"]["id"]: m for m in resp["data"]} + by_id = {m["model_info"]["id"]: m for m in json.loads(resp.body)["data"]} assert by_id["byok-id-1"]["model_info"]["access_via_team_ids"] == [team_id] assert by_id["byok-id-1"]["model_info"]["direct_access"] is False assert by_id["global-id-1"]["model_info"]["direct_access"] is True @@ -816,7 +818,7 @@ async def test_model_info_v1_litellm_model_id_include_team_models_filters_inacce include_team_models=True, ) - assert resp["data"] == [] + assert json.loads(resp.body)["data"] == [] @pytest.mark.asyncio @@ -852,7 +854,7 @@ async def test_model_info_v1_litellm_model_id_team_id_applies_team_filter(monkey teamId="other-team", ) - assert resp["data"] == [] + assert json.loads(resp.body)["data"] == [] team_filter.assert_awaited_once() assert team_filter.await_args.kwargs["team_id"] == "other-team" assert team_filter.await_args.kwargs["all_models"] == [team_row] diff --git a/tests/test_litellm/proxy/test_model_info_default_limits.py b/tests/test_litellm/proxy/test_model_info_default_limits.py index 8111a7af006..4426252cb6c 100644 --- a/tests/test_litellm/proxy/test_model_info_default_limits.py +++ b/tests/test_litellm/proxy/test_model_info_default_limits.py @@ -3,6 +3,7 @@ Tests verifying that default_api_key_tpm_limit and default_api_key_rpm_limit set litellm_params are returned by the /model/info endpoint. """ +import json from typing import Optional from unittest.mock import MagicMock, patch @@ -128,8 +129,9 @@ class TestModelInfoEndpointWithRouter: litellm_model_id="some-model-id", ) - assert len(response["data"]) == 1 - litellm_params = response["data"][0]["litellm_params"] + data = json.loads(response.body)["data"] + assert len(data) == 1 + litellm_params = data[0]["litellm_params"] assert litellm_params.get("default_api_key_tpm_limit") == 100 assert litellm_params.get("default_api_key_rpm_limit") == 200 @@ -171,7 +173,8 @@ class TestModelInfoEndpointWithRouter: litellm_model_id=None, ) - assert len(response["data"]) >= 1 - litellm_params = response["data"][0]["litellm_params"] + data = json.loads(response.body)["data"] + assert len(data) >= 1 + litellm_params = data[0]["litellm_params"] assert litellm_params.get("default_api_key_tpm_limit") == 100 assert litellm_params.get("default_api_key_rpm_limit") == 200 diff --git a/tests/test_litellm/proxy/test_model_list_healthy_only.py b/tests/test_litellm/proxy/test_model_list_healthy_only.py index 03eaa2e79c9..718c7e41da8 100644 --- a/tests/test_litellm/proxy/test_model_list_healthy_only.py +++ b/tests/test_litellm/proxy/test_model_list_healthy_only.py @@ -6,6 +6,7 @@ per-request `healthy_only` query parameter and the proxy-wide (`model_info_v1`). """ +import json from unittest.mock import AsyncMock, MagicMock import pytest @@ -275,7 +276,7 @@ async def test_model_info_v1_healthy_only_hides_unhealthy_deployments( litellm_model_id=None, healthy_only=True, ) - assert [m["model_name"] for m in response["data"]] == ["gpt-4"] + assert [m["model_name"] for m in json.loads(response.body)["data"]] == ["gpt-4"] @pytest.mark.asyncio @@ -286,7 +287,7 @@ async def test_model_info_v1_general_setting_hides_unhealthy_deployments(patched user_api_key_dict=_admin_key(), litellm_model_id=None, ) - assert [m["model_name"] for m in response["data"]] == ["gpt-4"] + assert [m["model_name"] for m in json.loads(response.body)["data"]] == ["gpt-4"] @pytest.mark.asyncio @@ -297,7 +298,7 @@ async def test_model_info_v1_default_keeps_unhealthy_deployments( user_api_key_dict=_admin_key(), litellm_model_id=None, ) - assert [m["model_name"] for m in response["data"]] == ["gpt-4", "claude-sonnet"] + assert [m["model_name"] for m in json.loads(response.body)["data"]] == ["gpt-4", "claude-sonnet"] patched_model_info_v1.async_get_fully_unhealthy_model_names.assert_not_awaited() @@ -318,4 +319,4 @@ async def test_model_info_v1_litellm_model_id_lookup_ignores_health_filter(patch user_api_key_dict=_admin_key(), litellm_model_id="unhealthy-id", ) - assert [m["model_name"] for m in response["data"]] == ["claude-sonnet"] + assert [m["model_name"] for m in json.loads(response.body)["data"]] == ["claude-sonnet"] diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 03a24ec5e98..a5e6cf285ad 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -15,10 +15,12 @@ from unittest import mock from unittest.mock import AsyncMock, MagicMock, create_autospec, mock_open, patch import click +import fastapi.routing import httpx import pytest import yaml from fastapi import FastAPI +from fastapi.encoders import jsonable_encoder from fastapi.staticfiles import StaticFiles from fastapi.testclient import TestClient @@ -5247,6 +5249,8 @@ async def test_model_info_v1_oci_secrets_not_leaked(): result = await model_info_v1(user_api_key_dict=mock_user_api_key_dict, litellm_model_id=None) # Verify the result structure + result_str = result.body.decode() + result = json.loads(result_str) assert "data" in result assert len(result["data"]) == 1 @@ -5269,13 +5273,96 @@ async def test_model_info_v1_oci_secrets_not_leaked(): assert litellm_params["model"].startswith("oci/"), "model should retain its full value" # Verify that actual secret values are not present in the response - result_str = str(result) assert "ocid1.api_key.oc1..aaaaaaaa7kbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbk" not in result_str assert "aa:bb:cc:dd:ee:ff:11:22:33:44:55:66:77:88:99:00" not in result_str assert "ocid1.tenancy.oc1..aaaaaaaa7kbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbk" not in result_str assert "/path/to/oci_api_key.pem" not in result_str +def test_model_info_v1_list_skips_fastapi_jsonable_encoder(monkeypatch): + """ + /model/info serializes its multi-megabyte listing itself with orjson. FastAPI must not + re-walk the payload through `jsonable_encoder`, while values orjson cannot encode natively + still come out as JSON. + """ + created_at = datetime(2026, 1, 2, 3, 4, 5, tzinfo=timezone.utc) + model_data = { + "model_name": "gpt-4o", + "litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-secret-value"}, + "model_info": { + "id": "db-row-1", + "db_model": True, + "created_at": created_at, + "supported_regions": frozenset({"eu"}), + }, + } + mock_router = MagicMock() + mock_router.model_list = [model_data] + mock_router.get_model_list_from_model_alias.return_value = [] + mock_router.get_model_names.return_value = ["gpt-4o"] + mock_router.get_model_access_groups.return_value = {} + mock_router.get_deployment.return_value = None + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", mock_router) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_model_list", [model_data]) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {"infer_model_from_keys": False}) + monkeypatch.setattr("litellm.proxy.proxy_server.user_model", None) + + encoder_spy = MagicMock(wraps=jsonable_encoder) + monkeypatch.setattr(fastapi.routing, "jsonable_encoder", encoder_spy) + + original_overrides = app.dependency_overrides.copy() + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-1234", models=[], team_models=[] + ) + client = TestClient(app) + try: + response = client.get("/model/info") + finally: + app.dependency_overrides = original_overrides + + assert response.status_code == 200 + assert response.headers["content-type"] == "application/json" + rows = response.json()["data"] + assert [row["model_name"] for row in rows] == ["gpt-4o"] + assert rows[0]["model_info"]["created_at"] == created_at.isoformat() + assert rows[0]["model_info"]["supported_regions"] == ["eu"] + assert "sk-secret-value" not in response.text + assert encoder_spy.call_count == 0 + + +def test_model_info_v1_cli_model_returns_single_deployment_as_json(monkeypatch): + """ + A proxy started with `litellm --model ` answers /model/info with one deployment + object under `data`, serialized the same way as the listing. + """ + monkeypatch.setattr("litellm.proxy.proxy_server.user_model", "gpt-4o") + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_model_list", None) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {}) + + encoder_spy = MagicMock(wraps=jsonable_encoder) + monkeypatch.setattr(fastapi.routing, "jsonable_encoder", encoder_spy) + + original_overrides = app.dependency_overrides.copy() + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-1234", models=[], team_models=[] + ) + client = TestClient(app) + try: + response = client.get("/model/info") + finally: + app.dependency_overrides = original_overrides + + assert response.status_code == 200 + assert response.headers["content-type"] == "application/json" + deployment = response.json()["data"] + assert deployment["model_name"] == "*" + assert deployment["litellm_params"]["model"] == "gpt-4o" + assert encoder_spy.call_count == 0 + + def test_add_callback_from_db_to_in_memory_litellm_callbacks(): """ Test that _add_callback_from_db_to_in_memory_litellm_callbacks correctly adds callbacks diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 8c4ed5d7ff5..0b0e3e18215 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -8938,7 +8938,7 @@ export interface paths { * `model_info.direct_access` when the proxy database is connected. * * Returns: - * Returns a dictionary containing information about each model. + * A JSON response whose `data` list holds one entry per model. * * Example Response: * ```json @@ -19084,7 +19084,7 @@ export interface paths { * `model_info.direct_access` when the proxy database is connected. * * Returns: - * Returns a dictionary containing information about each model. + * A JSON response whose `data` list holds one entry per model. * * Example Response: * ```json