scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-14 23:21:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 102

perf(proxy): serialize /model/info listing once with orjson

Browse source 
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
yassin 2026-09-14 19:47:51 +00:00
parent b94b8bca21
commit 39f6ac4788
7 changed files with 129 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
litellm/proxy/proxy_server.py
View file

@ -15036,6 +15036,13 @@ def _get_proxy_model_info(model: dict) -> dict:
return _translate_model_name_for_response(model)
def _model_info_json_response(data: Sequence[Mapping[str, object]] | Mapping[str, object]) -> Response:
return Response(
content=orjson.dumps({"data": data}, default=jsonable_encoder, option=orjson.OPT_NON_STR_KEYS),
media_type="application/json",
)
@router.get(
"/model/info",
tags=["model management"],
@ -15083,7 +15090,7 @@ async def model_info_v1(
`model_info.direct_access` when the proxy database is connected.
Returns:
Returns a dictionary containing information about each model.
A JSON response whose `data` list holds one entry per model.
Example Response:
```json
@ -15131,7 +15138,7 @@ async def model_info_v1(
deployment_dict=_deployment_info_dict,
excluded_keys={"litellm_credential_name"},
)
return {"data": _deployment_info_dict}
return _model_info_json_response(_deployment_info_dict)
if llm_model_list is None:
raise HTTPException(
@ -15182,7 +15189,7 @@ async def model_info_v1(
llm_router=llm_router,
user_api_key_dict=user_api_key_dict,
)
return {"data": single_model_list}
return _model_info_json_response(single_model_list)
# Return router deployments (same source as /v2/model/info), not wildcard-
# expanded model names from get_complete_model_list(). Team-scoped rows
@ -15250,7 +15257,7 @@ async def model_info_v1(
visible_models: Final = [model for model in all_models if model.get("model_name") not in hidden_names]
verbose_proxy_logger.debug("all_models: %s", visible_models)
return {"data": visible_models}
return _model_info_json_response(visible_models)
@router.get(

6
tests/proxy_unit_tests/test_proxy_server.py
View file

@ -2139,7 +2139,7 @@ async def test_model_info_alias_without_prisma(hidden):
user_api_key_dict=UserAPIKeyAuth(models=[]),
)
models = resp["data"]
models = json.loads(resp.body)["data"]
alias_found = any(
m["model_name"] == model_alias
@ -2203,7 +2203,7 @@ async def test_proxy_model_group_alias_checks(prisma_client, hidden): # noqa: F
resp = await model_info_v1(
user_api_key_dict=UserAPIKeyAuth(models=[]),
)
models = resp["data"]
models = json.loads(resp.body)["data"]
is_model_alias_in_list = False
for item in models:
if model_alias == item["model_name"]:
@ -2280,7 +2280,7 @@ async def test_proxy_model_group_info_rerank(prisma_client): # noqa: F811 # py
resp = await model_info_v1(
user_api_key_dict=UserAPIKeyAuth(models=[]),
)
models = resp["data"]
models = json.loads(resp.body)["data"]
assert models[0]["model_info"]["mode"] == "rerank"
resp = await model_group_info(
user_api_key_dict=UserAPIKeyAuth(models=[]),

24
tests/test_litellm/proxy/proxy_server/test_team_model_name_translation.py
View file

@ -9,6 +9,7 @@ rows instead of the internal routing key `model_name_{team_id}_{uuid}`.
from __future__ import annotations
import json
from unittest.mock import AsyncMock, MagicMock
import pytest
@ -238,7 +239,7 @@ async def test_model_info_v1_list_path_translates_team_model_name(monkeypatch):
)
resp = await ps.model_info_v1(user_api_key_dict=admin, litellm_model_id=None)
names = [m["model_name"] for m in resp["data"]]
names = [m["model_name"] for m in json.loads(resp.body)["data"]]
assert "team-claude-sonnet" in names
assert "model_name_team-abc-123_4a6b8" not in names
@ -271,7 +272,7 @@ async def test_model_info_v1_unrestricted_key_returns_all_deployments(monkeypatc
)
resp = await ps.model_info_v1(user_api_key_dict=caller, litellm_model_id=None)
assert [m["model_name"] for m in resp["data"]] == ["gpt-4"]
assert [m["model_name"] for m in json.loads(resp.body)["data"]] == ["gpt-4"]
@pytest.mark.asyncio
@ -303,7 +304,7 @@ async def test_model_info_v1_restricted_key_filters_deployments(monkeypatch):
)
resp = await ps.model_info_v1(user_api_key_dict=caller, litellm_model_id=None)
assert [m["model_name"] for m in resp["data"]] == ["gpt-4"]
assert [m["model_name"] for m in json.loads(resp.body)["data"]] == ["gpt-4"]
def _other_team_row() -> dict:
@ -367,10 +368,11 @@ async def test_model_info_v1_unrestricted_key_hides_other_team_byok(monkeypatch)
)
resp = await ps.model_info_v1(user_api_key_dict=caller, litellm_model_id=None)
returned_ids = {m["model_info"]["id"] for m in resp["data"]}
data = json.loads(resp.body)["data"]
returned_ids = {m["model_info"]["id"] for m in data}
assert returned_ids == {"global-id-1", "byok-id-1"}
assert "byok-id-other" not in returned_ids
names = [m["model_name"] for m in resp["data"]]
names = [m["model_name"] for m in data]
assert "team-claude-sonnet" in names
assert "gpt-4" in names
@ -412,7 +414,7 @@ async def test_model_info_v1_service_key_hides_all_team_byok(monkeypatch):
)
resp = await ps.model_info_v1(user_api_key_dict=caller, litellm_model_id=None)
assert [m["model_info"]["id"] for m in resp["data"]] == ["global-id-1"]
assert [m["model_info"]["id"] for m in json.loads(resp.body)["data"]] == ["global-id-1"]
@pytest.mark.asyncio
@ -466,7 +468,7 @@ async def test_model_info_v1_team_key_sees_own_byok_regardless_of_user_lookup(
)
resp = await ps.model_info_v1(user_api_key_dict=caller, litellm_model_id=None)
assert [m["model_info"]["id"] for m in resp["data"]] == ["byok-id-1", "global-id-1"]
assert [m["model_info"]["id"] for m in json.loads(resp.body)["data"]] == ["byok-id-1", "global-id-1"]
@pytest.mark.asyncio
@ -509,7 +511,7 @@ async def test_model_info_v1_user_team_membership_grants_byok(monkeypatch):
)
resp = await ps.model_info_v1(user_api_key_dict=caller, litellm_model_id=None)
assert [m["model_info"]["id"] for m in resp["data"]] == [
assert [m["model_info"]["id"] for m in json.loads(resp.body)["data"]] == [
"byok-id-other",
"global-id-1",
]
@ -557,7 +559,7 @@ async def test_model_info_v1_populates_access_via_team_ids(monkeypatch):
)
resp = await ps.model_info_v1(user_api_key_dict=admin, litellm_model_id=None)
by_id = {m["model_info"]["id"]: m for m in resp["data"]}
by_id = {m["model_info"]["id"]: m for m in json.loads(resp.body)["data"]}
assert by_id["byok-id-1"]["model_info"]["access_via_team_ids"] == [team_id]
assert by_id["byok-id-1"]["model_info"]["direct_access"] is False
assert by_id["global-id-1"]["model_info"]["direct_access"] is True
@ -816,7 +818,7 @@ async def test_model_info_v1_litellm_model_id_include_team_models_filters_inacce
include_team_models=True,
)
assert resp["data"] == []
assert json.loads(resp.body)["data"] == []
@pytest.mark.asyncio
@ -852,7 +854,7 @@ async def test_model_info_v1_litellm_model_id_team_id_applies_team_filter(monkey
teamId="other-team",
)
assert resp["data"] == []
assert json.loads(resp.body)["data"] == []
team_filter.assert_awaited_once()
assert team_filter.await_args.kwargs["team_id"] == "other-team"
assert team_filter.await_args.kwargs["all_models"] == [team_row]

11
tests/test_litellm/proxy/test_model_info_default_limits.py
View file

@ -3,6 +3,7 @@ Tests verifying that default_api_key_tpm_limit and default_api_key_rpm_limit set
litellm_params are returned by the /model/info endpoint.
"""
import json
from typing import Optional
from unittest.mock import MagicMock, patch
@ -128,8 +129,9 @@ class TestModelInfoEndpointWithRouter:
litellm_model_id="some-model-id",
)
assert len(response["data"]) == 1
litellm_params = response["data"][0]["litellm_params"]
data = json.loads(response.body)["data"]
assert len(data) == 1
litellm_params = data[0]["litellm_params"]
assert litellm_params.get("default_api_key_tpm_limit") == 100
assert litellm_params.get("default_api_key_rpm_limit") == 200
@ -171,7 +173,8 @@ class TestModelInfoEndpointWithRouter:
litellm_model_id=None,
)
assert len(response["data"]) >= 1
litellm_params = response["data"][0]["litellm_params"]
data = json.loads(response.body)["data"]
assert len(data) >= 1
litellm_params = data[0]["litellm_params"]
assert litellm_params.get("default_api_key_tpm_limit") == 100
assert litellm_params.get("default_api_key_rpm_limit") == 200

9
tests/test_litellm/proxy/test_model_list_healthy_only.py
View file

@ -6,6 +6,7 @@ per-request `healthy_only` query parameter and the proxy-wide
(`model_info_v1`).
"""
import json
from unittest.mock import AsyncMock, MagicMock
import pytest
@ -275,7 +276,7 @@ async def test_model_info_v1_healthy_only_hides_unhealthy_deployments(
litellm_model_id=None,
healthy_only=True,
)
assert [m["model_name"] for m in response["data"]] == ["gpt-4"]
assert [m["model_name"] for m in json.loads(response.body)["data"]] == ["gpt-4"]
@pytest.mark.asyncio
@ -286,7 +287,7 @@ async def test_model_info_v1_general_setting_hides_unhealthy_deployments(patched
user_api_key_dict=_admin_key(),
litellm_model_id=None,
)
assert [m["model_name"] for m in response["data"]] == ["gpt-4"]
assert [m["model_name"] for m in json.loads(response.body)["data"]] == ["gpt-4"]
@pytest.mark.asyncio
@ -297,7 +298,7 @@ async def test_model_info_v1_default_keeps_unhealthy_deployments(
user_api_key_dict=_admin_key(),
litellm_model_id=None,
)
assert [m["model_name"] for m in response["data"]] == ["gpt-4", "claude-sonnet"]
assert [m["model_name"] for m in json.loads(response.body)["data"]] == ["gpt-4", "claude-sonnet"]
patched_model_info_v1.async_get_fully_unhealthy_model_names.assert_not_awaited()
@ -318,4 +319,4 @@ async def test_model_info_v1_litellm_model_id_lookup_ignores_health_filter(patch
user_api_key_dict=_admin_key(),
litellm_model_id="unhealthy-id",
)
assert [m["model_name"] for m in response["data"]] == ["claude-sonnet"]
assert [m["model_name"] for m in json.loads(response.body)["data"]] == ["claude-sonnet"]

89
tests/test_litellm/proxy/test_proxy_server.py
View file

@ -15,10 +15,12 @@ from unittest import mock
from unittest.mock import AsyncMock, MagicMock, create_autospec, mock_open, patch
import click
import fastapi.routing
import httpx
import pytest
import yaml
from fastapi import FastAPI
from fastapi.encoders import jsonable_encoder
from fastapi.staticfiles import StaticFiles
from fastapi.testclient import TestClient
@ -5247,6 +5249,8 @@ async def test_model_info_v1_oci_secrets_not_leaked():
result = await model_info_v1(user_api_key_dict=mock_user_api_key_dict, litellm_model_id=None)
# Verify the result structure
result_str = result.body.decode()
result = json.loads(result_str)
assert "data" in result
assert len(result["data"]) == 1
@ -5269,13 +5273,96 @@ async def test_model_info_v1_oci_secrets_not_leaked():
assert litellm_params["model"].startswith("oci/"), "model should retain its full value"
# Verify that actual secret values are not present in the response
result_str = str(result)
assert "ocid1.api_key.oc1..aaaaaaaa7kbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbk" not in result_str
assert "aa:bb:cc:dd:ee:ff:11:22:33:44:55:66:77:88:99:00" not in result_str
assert "ocid1.tenancy.oc1..aaaaaaaa7kbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbkbk" not in result_str
assert "/path/to/oci_api_key.pem" not in result_str
def test_model_info_v1_list_skips_fastapi_jsonable_encoder(monkeypatch):
"""
/model/info serializes its multi-megabyte listing itself with orjson. FastAPI must not
re-walk the payload through `jsonable_encoder`, while values orjson cannot encode natively
still come out as JSON.
"""
created_at = datetime(2026, 1, 2, 3, 4, 5, tzinfo=timezone.utc)
model_data = {
"model_name": "gpt-4o",
"litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-secret-value"},
"model_info": {
"id": "db-row-1",
"db_model": True,
"created_at": created_at,
"supported_regions": frozenset({"eu"}),
},
}
mock_router = MagicMock()
mock_router.model_list = [model_data]
mock_router.get_model_list_from_model_alias.return_value = []
mock_router.get_model_names.return_value = ["gpt-4o"]
mock_router.get_model_access_groups.return_value = {}
mock_router.get_deployment.return_value = None
monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", mock_router)
monkeypatch.setattr("litellm.proxy.proxy_server.llm_model_list", [model_data])
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None)
monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {"infer_model_from_keys": False})
monkeypatch.setattr("litellm.proxy.proxy_server.user_model", None)
encoder_spy = MagicMock(wraps=jsonable_encoder)
monkeypatch.setattr(fastapi.routing, "jsonable_encoder", encoder_spy)
original_overrides = app.dependency_overrides.copy()
app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(
user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-1234", models=[], team_models=[]
)
client = TestClient(app)
try:
response = client.get("/model/info")
finally:
app.dependency_overrides = original_overrides
assert response.status_code == 200
assert response.headers["content-type"] == "application/json"
rows = response.json()["data"]
assert [row["model_name"] for row in rows] == ["gpt-4o"]
assert rows[0]["model_info"]["created_at"] == created_at.isoformat()
assert rows[0]["model_info"]["supported_regions"] == ["eu"]
assert "sk-secret-value" not in response.text
assert encoder_spy.call_count == 0
def test_model_info_v1_cli_model_returns_single_deployment_as_json(monkeypatch):
"""
A proxy started with `litellm --model <name>` answers /model/info with one deployment
object under `data`, serialized the same way as the listing.
"""
monkeypatch.setattr("litellm.proxy.proxy_server.user_model", "gpt-4o")
monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None)
monkeypatch.setattr("litellm.proxy.proxy_server.llm_model_list", None)
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None)
monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {})
encoder_spy = MagicMock(wraps=jsonable_encoder)
monkeypatch.setattr(fastapi.routing, "jsonable_encoder", encoder_spy)
original_overrides = app.dependency_overrides.copy()
app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(
user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-1234", models=[], team_models=[]
)
client = TestClient(app)
try:
response = client.get("/model/info")
finally:
app.dependency_overrides = original_overrides
assert response.status_code == 200
assert response.headers["content-type"] == "application/json"
deployment = response.json()["data"]
assert deployment["model_name"] == "*"
assert deployment["litellm_params"]["model"] == "gpt-4o"
assert encoder_spy.call_count == 0
def test_add_callback_from_db_to_in_memory_litellm_callbacks():
"""
Test that _add_callback_from_db_to_in_memory_litellm_callbacks correctly adds callbacks

4
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -8938,7 +8938,7 @@ export interface paths {
* `model_info.direct_access` when the proxy database is connected.
*
* Returns:
* Returns a dictionary containing information about each model.
* A JSON response whose `data` list holds one entry per model.
*
* Example Response:
* ```json
@ -19084,7 +19084,7 @@ export interface paths {
* `model_info.direct_access` when the proxy database is connected.
*
* Returns:
* Returns a dictionary containing information about each model.
* A JSON response whose `data` list holds one entry per model.
*
* Example Response:
* ```json