scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-11 22:51:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 124

Merge remote-tracking branch 'origin/litellm_internal_staging' into litellm_registry_audit_2026_09_02

Browse source
This commit is contained in:
mateo 2026-09-04 13:03:22 +00:00
commit af4340dce3
15 changed files with 1238 additions and 367 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
litellm/proxy/_types.py
View file

@ -737,6 +737,9 @@ class LiteLLMRoutes(enum.Enum):
"/.well-known/litellm-ui-config",
"/public/model_hub",
"/public/v1/model_hub",
"/public/v1/model_hub/providers",
"/public/v1/model_hub/modes",
"/public/v1/model_hub/features",
"/public/model_hub/info",
"/public/agent_hub",
"/public/mcp_hub",

12
litellm/proxy/list_api/in_memory.py
View file

@ -141,3 +141,15 @@ class InMemoryListExecutor(Generic[TRow]):
async def find_many(self, plan: QueryPlan) -> Sequence[TRow]:
page: Final = _ordered(self._matching(plan.where), plan.order)[plan.skip : plan.skip + plan.take]
return await self.enrich_page(tuple(row for _, row in page))
async def distinct(self, field: str, where: tuple[Predicate, ...]) -> Sequence[str]:
"""A repeated field contributes each of its elements, so a facet over `providers`
lists providers rather than the tuples rows happen to carry."""
cells: Final = (cells.get(field) for cells, _ in self._matching(where))
values: Final = (
value
for cell in cells
for value in (cell if isinstance(cell, tuple) else (cell,))
if isinstance(value, str) and value
)
return tuple(sorted(frozenset(values)))

82
litellm/proxy/list_api/list_framework.py
View file

@ -28,12 +28,15 @@ from litellm.proxy.list_api.common import (
PROBLEM_TYPE_BASE,
ManagementProblem,
build_list_links,
build_page_links,
escape_like,
unknown_query_param_problem,
)
from litellm.types.proxy.management_endpoints.management_v1 import (
FacetListResponse,
ListMeta,
ListResponse,
PageMeta,
ProblemDetail,
)
@ -186,6 +189,13 @@ class ListExecutor(Protocol[TRow_co]):
async def find_many(self, plan: QueryPlan) -> Sequence[TRow_co]: ...
class FacetExecutor(Protocol):
"""The half of a facet that knows the rows. Separate from `ListExecutor` so a SQL
executor is not forced to implement `distinct` to keep serving entity lists."""
async def distinct(self, field: str, where: tuple[Predicate, ...]) -> Sequence[str]: ...
def order_by_sql(order: tuple[SortKey, ...]) -> str:
"""`ORDER BY` body for a plan, NULLS LAST in both directions.
@ -515,6 +525,78 @@ def build_query_plan(
)
def _facet_allowed_params(spec: ListSpec[TRow, TOut]) -> tuple[str, ...]:
"""A facet's values are always ascending, so `sort` is not one of its parameters."""
return tuple(name for name in _allowed_params(spec) if name != SORT_PARAM)
def _facet_where(
spec: ListSpec[TRow, TOut],
params: Mapping[str, str],
caller: UserAPIKeyAuth,
) -> tuple[Predicate, ...] | ProblemDetail:
scope_predicates: Final = _scope_predicates(spec.scope(caller))
if isinstance(scope_predicates, ProblemDetail):
return scope_predicates
filters: Final = _parse_filters(spec, params)
if isinstance(filters, ProblemDetail):
return filters
search: Final = _search_predicate(spec, params)
return scope_predicates + filters + ((search,) if search is not None else ())
async def handle_facet(
spec: ListSpec[TRow, TOut],
executor: FacetExecutor,
request: Request,
caller: UserAPIKeyAuth,
field: str,
) -> FacetListResponse:
"""The distinct values one column takes over a filtered query on a resource.
Carries the parent's parameters so a filter dropdown offers exactly the values the
table can show, and `has_more` rather than a total, which would cost a COUNT(*) over
the whole match set on every keystroke.
"""
params: Final = request.query_params
unknown: Final = tuple(sorted(name for name in params if name == SORT_PARAM or not _is_known_param(spec, name)))
if unknown:
raise ManagementProblem(unknown_query_param_problem(unknown=unknown, allowed=_facet_allowed_params(spec)))
duplicates: Final = _duplicate_params(request)
if duplicates:
raise ManagementProblem(
_problem(
"duplicate-query-parameter",
"Duplicate query parameter",
400,
f"Repeated query parameter(s): {', '.join(duplicates)}. Each may appear once; "
f"use a comma-separated list for multiple filter values.",
)
)
page: Final = _parse_page(params)
if isinstance(page, ProblemDetail):
raise ManagementProblem(page)
page_size: Final = _parse_page_size(spec, params)
if isinstance(page_size, ProblemDetail):
raise ManagementProblem(page_size)
where: Final = _facet_where(spec, params, caller)
if isinstance(where, ProblemDetail):
raise ManagementProblem(where)
values: Final = await executor.distinct(field, where)
skip: Final = (page - 1) * page_size
window: Final = values[skip : skip + page_size + 1]
has_more: Final = len(window) > page_size
return FacetListResponse(
data=tuple(window[:page_size]),
meta=PageMeta(page=page, page_size=page_size, has_more=has_more),
links=build_page_links(request=request, page=page, has_more=has_more),
)
def _duplicate_params(request: Request) -> tuple[str, ...]:
names: Final = tuple(name for name, _ in request.query_params.multi_items())
return tuple(sorted(frozenset(name for name in names if names.count(name) > 1)))

141
litellm/proxy/public_endpoints/public_v1/model_hub.py
View file

@ -3,7 +3,7 @@
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from types import MappingProxyType
from typing import Annotated, Final, Protocol
from typing import Annotated, Final, Literal, Protocol
from fastapi import APIRouter, Depends, Request
from typing_extensions import ReadOnly, TypedDict
@ -20,10 +20,12 @@ from litellm.proxy.list_api.list_framework import (
Scope,
ScopeAll,
SortKey,
handle_facet,
handle_list,
)
from litellm.proxy.utils import PrismaClient
from litellm.types.proxy.management_endpoints.management_v1 import (
FacetListResponse,
ListResponse,
ProblemDetail,
)
@ -95,16 +97,37 @@ class HealthEnricher:
return tuple(_with_health(row, health.get(row.model_group)) for row in rows)
FEATURE_PREFIX: Final = "supports_"
def _features(row: ModelGroupInfoProxy) -> tuple[str, ...]:
"""A row's capabilities as one repeated field, so selecting two of them matches either.
The hub's feature control has always been a multi-select over the `supports_*` flags.
One boolean filter per flag would AND them, which is the opposite of what it does.
"""
return tuple(
sorted(
name.removeprefix(FEATURE_PREFIX)
for name, value in row.model_dump().items()
if name.startswith(FEATURE_PREFIX) and value is True
)
)
def _cells(row: ModelGroupInfoProxy) -> Cells:
return MappingProxyType(
{
"model_group": row.model_group,
"mode": row.mode,
"providers": tuple(row.providers),
"features": _features(row),
"max_input_tokens": row.max_input_tokens,
"max_output_tokens": row.max_output_tokens,
"input_cost_per_token": row.input_cost_per_token,
"output_cost_per_token": row.output_cost_per_token,
"rpm": row.rpm,
"tpm": row.tpm,
}
)
@ -126,20 +149,28 @@ def _scope(_caller: UserAPIKeyAuth) -> Scope:
MODEL_HUB_FILTERS: Final[Mapping[str, FilterSpec]] = MappingProxyType(
{
"mode": FilterSpec(type=str, ops=frozenset(("eq", "in"))),
"providers": FilterSpec(type=str, ops=frozenset(("contains",))),
"providers": FilterSpec(type=str, ops=frozenset(("contains", "in"))),
"features": FilterSpec(type=str, ops=frozenset(("in",))),
}
)
MODEL_HUB_FACETS: Final[Mapping[str, str]] = MappingProxyType(
{"providers": "providers", "modes": "mode", "features": "features"}
)
MODEL_HUB_LIST_SPEC: Final[ListSpec[ModelGroupInfoProxy, ModelGroupInfoProxy]] = ListSpec(
resource="model groups",
sortable=frozenset(
(
"model_group",
"mode",
"providers",
"max_input_tokens",
"max_output_tokens",
"input_cost_per_token",
"output_cost_per_token",
"rpm",
"tpm",
)
),
searchable=frozenset(("model_group",)),
@ -153,6 +184,32 @@ MODEL_HUB_LIST_SPEC: Final[ListSpec[ModelGroupInfoProxy, ModelGroupInfoProxy]] =
)
def _published_rows() -> Sequence[ModelGroupInfoProxy]:
from litellm.proxy.proxy_server import (
_get_model_group_info, # pyright: ignore[reportPrivateUsage] # /public/model_hub imports it the same way
llm_router,
)
if llm_router is None:
raise ManagementProblem(
ProblemDetail(
type=f"{PROBLEM_TYPE_BASE}no-llm-router",
title="No models configured",
status=400,
detail=CommonProxyErrors.no_llm_router.value,
)
)
if litellm.public_model_groups is None:
return ()
return tuple(
_get_model_group_info(
llm_router=llm_router,
all_models_str=litellm.public_model_groups,
model_group=None,
)
)
def _executor(
rows: Sequence[ModelGroupInfoProxy],
prisma_client: PrismaClient | None,
@ -191,37 +248,11 @@ async def public_model_hub_list(
```
"""
try:
from litellm.proxy.proxy_server import (
_get_model_group_info, # pyright: ignore[reportPrivateUsage] # /public/model_hub imports it the same way
llm_router,
prisma_client,
)
if llm_router is None:
raise ManagementProblem(
ProblemDetail(
type=f"{PROBLEM_TYPE_BASE}no-llm-router",
title="No models configured",
status=400,
detail=CommonProxyErrors.no_llm_router.value,
)
)
rows: Final[Sequence[ModelGroupInfoProxy]] = (
()
if litellm.public_model_groups is None
else tuple(
_get_model_group_info(
llm_router=llm_router,
all_models_str=litellm.public_model_groups,
model_group=None,
)
)
)
from litellm.proxy.proxy_server import prisma_client
return await handle_list(
spec=MODEL_HUB_LIST_SPEC,
executor=_executor(rows, prisma_client),
executor=_executor(_published_rows(), prisma_client),
request=request,
caller=user_api_key_dict,
)
@ -240,3 +271,53 @@ async def public_model_hub_list(
detail="Failed to list public model groups.",
)
)
@router.get(
"/model_hub/{facet}",
tags=["public", "model management"], # mutable-ok: fastapi types tags as list[str | Enum]
dependencies=(Depends(user_api_key_auth),),
response_model=FacetListResponse,
)
async def public_model_hub_facet(
request: Request,
facet: Literal["providers", "modes", "features"],
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
) -> FacetListResponse:
"""
The distinct providers, modes or features across the published model groups, for the
Model Hub's filter dropdowns. No authentication.
Carries the same filters and search as the list route, so a dropdown offers exactly
the values the table can show: asking for providers under `filter[mode][in]=chat`
lists only the providers that serve a chat model.
Example curl:
```
curl --location --globoff \
'http://0.0.0.0:4000/public/v1/model_hub/providers?filter[mode][in]=chat&page_size=50'
```
"""
try:
return await handle_facet(
spec=MODEL_HUB_LIST_SPEC,
executor=InMemoryListExecutor(rows=_published_rows(), cells=_cells),
request=request,
caller=user_api_key_dict,
field=MODEL_HUB_FACETS[facet],
)
except ManagementProblem:
raise
except Exception as e: # noqa: BLE001 # a router error answers as a problem document, not the OpenAI error shape
verbose_proxy_logger.exception(
"litellm.proxy.public_endpoints.public_v1.model_hub.public_model_hub_facet(): Exception occured - %s", e
)
raise ManagementProblem(
ProblemDetail(
type=f"{PROBLEM_TYPE_BASE}internal-server-error",
title="Internal server error",
status=500,
detail="Failed to list public model group values.",
)
)

3
litellm/types/proxy/management_endpoints/management_v1.py
View file

@ -1,5 +1,6 @@
"""Shared response shapes for the `/management/v1` control-plane surface."""
from collections.abc import Sequence
from typing import Generic, TypeVar
from pydantic import BaseModel, ConfigDict, Field
@ -38,7 +39,7 @@ class PageMeta(BaseModel):
class FacetListResponse(BaseModel):
"""The distinct values one column takes over a filtered query. `data` holds bare values, not entity rows."""
data: list[str]
data: Sequence[str]
meta: PageMeta
links: PageLinks

105
tests/test_litellm/proxy/list_api/test_list_framework.py
View file

@ -25,6 +25,7 @@ from litellm.proxy.list_api.list_framework import (
SortKey,
Within,
build_query_plan,
handle_facet,
handle_list,
order_by_sql,
where_sql,
@ -892,3 +893,107 @@ def test_the_facet_page_shapes_are_untouched_by_page_mode():
assert set(links) == {"self", "prev", "next"}
assert links["next"] == "/management/v1/budgets?q=ac&page=3"
# ------------------------------------------------------- facet request handling
class RecordingFacetExecutor:
"""Records the one call `handle_facet` is allowed to make, so a rejected request
can be shown never to have reached it."""
def __init__(self, values: tuple[str, ...] = ()) -> None:
self.values = values
self.field: str | None = None
self.where: tuple[object, ...] | None = None
async def distinct(self, field: str, where: tuple[object, ...]) -> Sequence[str]:
self.field = field
self.where = where
return self.values
async def _facet_problem(query: str, spec: ListSpec[BudgetRow, BudgetOut] | None = None) -> ProblemDetail:
executor = RecordingFacetExecutor(values=("a", "b"))
with pytest.raises(ManagementProblem) as raised:
await handle_facet(
spec=spec or _spec(),
executor=executor,
request=_request(query),
caller=CALLER,
field="created_by",
)
assert executor.field is None, "a rejected facet request still queried the executor"
return raised.value.problem
@pytest.mark.asyncio
async def test_a_facet_conjoins_the_scope_with_the_callers_filters():
"""The scope is the one predicate a caller cannot drop, so a facet has to add to it
rather than replace it: otherwise a dropdown lists values from rows the caller
cannot see in the table."""
spec = _spec(scope=lambda caller: ScopeWhere(where=(Compare(field="created_by", op="eq", value="caller-1"),)))
executor = RecordingFacetExecutor(values=("caller-1",))
response = await handle_facet(
spec=spec,
executor=executor,
request=_request("filter[max_budget][gte]=5&q=ac"),
caller=CALLER,
field="created_by",
)
assert tuple(response.data) == ("caller-1",)
assert executor.where == (
Compare(field="created_by", op="eq", value="caller-1"),
Compare(field="max_budget", op="gte", value=5.0),
AnyOf(
clauses=(
Compare(field="budget_id", op="contains", value="ac"),
Compare(field="created_by", op="contains", value="ac"),
)
),
)
@pytest.mark.asyncio
async def test_a_denied_scope_on_a_facet_never_reaches_the_executor():
"""A 200 with an empty list would read as "no such values" rather than "not yours"."""
problem = await _facet_problem("", spec=_spec(scope=lambda caller: ScopeDenied(reason="nope")))
assert problem.status == 403
assert problem.type == f"{PROBLEM_TYPE_BASE}forbidden"
@pytest.mark.asyncio
async def test_a_facet_rejects_a_filter_operator_its_spec_does_not_offer():
problem = await _facet_problem("filter[created_by][gte]=x")
assert problem.status == 400
assert "gte" in problem.detail
@pytest.mark.asyncio
async def test_a_facet_rejects_a_repeated_query_parameter():
problem = await _facet_problem("page=1&page=2")
assert problem.type == f"{PROBLEM_TYPE_BASE}duplicate-query-parameter"
assert "page" in problem.detail
@pytest.mark.asyncio
@pytest.mark.parametrize("query", ("page=0", "page=one"))
async def test_a_facet_rejects_a_page_that_is_not_a_positive_integer(query: str):
problem = await _facet_problem(query)
assert problem.status == 400
assert problem.type == f"{PROBLEM_TYPE_BASE}invalid-query-parameter"
assert "'page'" in problem.detail
@pytest.mark.asyncio
async def test_a_facet_rejects_a_page_size_that_is_not_a_positive_integer():
problem = await _facet_problem("page_size=0")
assert problem.status == 400
assert "'page_size'" in problem.detail

135
tests/test_litellm/proxy/public_endpoints/public_v1/test_model_hub.py
View file

@ -193,12 +193,12 @@ def test_sorting_by_a_numeric_field_puts_the_unset_ones_last_in_both_directions(
def test_an_undeclared_sort_field_is_a_problem_naming_the_allowed_fields(monkeypatch):
_publish(monkeypatch, _named(3))
response = _get("sort=providers")
response = _get("sort=health_status")
assert response.status_code == 400
assert response.headers["content-type"].startswith("application/problem+json")
body = response.json()
assert "providers" in body["detail"]
assert "health_status" in body["detail"]
assert body["allowed"] == [
"input_cost_per_token",
"max_input_tokens",
@ -206,6 +206,9 @@ def test_an_undeclared_sort_field_is_a_problem_naming_the_allowed_fields(monkeyp
"mode",
"model_group",
"output_cost_per_token",
"providers",
"rpm",
"tpm",
]
@ -347,3 +350,131 @@ def test_the_endpoint_it_supersedes_still_answers_with_its_bare_array(monkeypatc
body = response.json()
assert isinstance(body, list)
assert [row["model_group"] for row in body] == ["model-000", "model-001", "model-002"]
FACET_PATHS = ("providers", "modes", "features")
def _facet(name: str, query: str = ""):
suffix = f"?{query}" if query else ""
return client.get(f"{MODEL_HUB_PATH}/{name}{suffix}")
def test_providers_filter_accepts_several_providers_at_once(monkeypatch):
"""The hub's provider control is a multi-select, so the route has to OR the values."""
_publish(
monkeypatch,
(
_info("gpt-4", providers=("openai",)),
_info("claude", providers=("anthropic",)),
_info("mistral-large", providers=("mistral",)),
_info("router", providers=("openai", "anthropic")),
),
)
response = _get("filter[providers][in]=openai,anthropic")
assert response.status_code == 200, response.text
assert sorted(_groups(response)) == ["claude", "gpt-4", "router"]
def test_features_filter_matches_a_model_with_any_of_the_named_features(monkeypatch):
"""Selecting two features widens the result set, the way the hub's multi-select always did."""
_publish(
monkeypatch,
(
_info("sees", supports_vision=True),
_info("calls", supports_function_calling=True),
_info("both", supports_vision=True, supports_function_calling=True),
_info("plain"),
),
)
response = _get("filter[features][in]=vision,function_calling")
assert response.status_code == 200, response.text
assert sorted(_groups(response)) == ["both", "calls", "sees"]
def test_a_single_feature_filter_selects_only_models_with_it(monkeypatch):
_publish(monkeypatch, (_info("sees", supports_vision=True), _info("plain"), _info("reasons", supports_reasoning=True)))
assert _groups(_get("filter[features][in]=vision")) == ["sees"]
assert _groups(_get("filter[features][in]=reasoning")) == ["reasons"]
def test_providers_and_limits_are_sortable(monkeypatch):
"""The hub sorted on these columns before it paged; they stay sortable now that the route orders."""
_publish(
monkeypatch,
(
_info("b-model", providers=("mistral",), rpm=10),
_info("a-model", providers=("anthropic",), rpm=30),
_info("c-model", providers=("openai",), rpm=20),
),
)
assert _groups(_get("sort=providers")) == ["a-model", "b-model", "c-model"]
assert _groups(_get("sort=-rpm")) == ["a-model", "c-model", "b-model"]
@pytest.mark.parametrize("facet", FACET_PATHS)
def test_a_facet_serves_the_distinct_values_of_its_column(monkeypatch, facet):
_publish(
monkeypatch,
(
_info("a", providers=("openai",), mode="chat", supports_vision=True),
_info("b", providers=("anthropic", "openai"), mode="embedding", supports_vision=True),
_info("c", providers=("mistral",), mode="chat"),
),
)
response = _facet(facet)
assert response.status_code == 200, response.text
assert response.json()["data"] == {
"providers": ["anthropic", "mistral", "openai"],
"modes": ["chat", "embedding"],
"features": ["vision"],
}[facet]
def test_a_facet_offers_only_values_the_table_can_show(monkeypatch):
"""Section 12's reason for hanging facets off the resource: the dropdown matches the filtered table."""
_publish(
monkeypatch,
(
_info("chat-openai", providers=("openai",), mode="chat"),
_info("embed-cohere", providers=("cohere",), mode="embedding"),
),
)
assert _facet("providers", "filter[mode][in]=chat").json()["data"] == ["openai"]
assert _facet("providers", "q=embed").json()["data"] == ["cohere"]
def test_a_facet_pages_and_reports_whether_more_remain(monkeypatch):
_publish(monkeypatch, tuple(_info(f"m-{index}", providers=(f"p-{index:02d}",)) for index in range(5)))
first = _facet("providers", "page_size=2")
last = _facet("providers", "page=3&page_size=2")
assert first.json()["data"] == ["p-00", "p-01"]
assert first.json()["meta"] == {"page": 1, "page_size": 2, "has_more": True}
assert last.json()["data"] == ["p-04"]
assert last.json()["meta"]["has_more"] is False
def test_a_facet_rejects_a_sort_it_does_not_offer(monkeypatch):
"""Facet values are always ascending, so `sort` is not part of the facet contract."""
_publish(monkeypatch, _named(3))
response = _facet("providers", "sort=-providers")
assert response.status_code == 400
assert response.json()["type"].endswith("unknown-query-parameter")
@pytest.mark.parametrize("facet", FACET_PATHS)
def test_a_facet_is_reachable_without_a_key(facet):
assert f"{MODEL_HUB_PATH}/{facet}" in LiteLLMRoutes.public_routes.value

289
ui/litellm-dashboard/src/components/PublicModelHubTableColumns.tsx
View file

@ -6,6 +6,7 @@ import { DataTableSortHeader } from "@/components/shared/DataTable";
import { CellTooltip, IdentityCell, StatusBadge, type StatusTone } from "@/components/shared/table_cells";
import { Badge } from "@/components/ui/badge";
import { getProviderLogoAndName } from "@/components/provider_info_helpers";
import { PUBLIC_MODEL_HUB_SORTABLE_FIELDS } from "@/components/publicModelHub/publicModelHubFilters";
export interface ModelGroupInfo {
model_group: string;
@ -163,154 +164,150 @@ interface PublicModelHubColumnsDeps {
onModelClick: (model: ModelGroupInfo) => void;
}
export const getPublicModelHubColumns = ({ onModelClick }: PublicModelHubColumnsDeps): ColumnDef<ModelGroupInfo>[] => [
{
id: "model_group",
accessorKey: "model_group",
meta: { title: "Model Name" },
header: ({ column }) => <DataTableSortHeader column={column} title="Model Name" />,
size: 200,
enableSorting: true,
sortingFn: "alphanumeric",
cell: ({ row }) => (
<IdentityCell
title={row.original.model_group}
titleClassName="font-mono text-xs font-normal"
className="max-w-72"
onClick={() => onModelClick(row.original)}
/>
),
},
{
id: "providers",
accessorKey: "providers",
meta: { title: "Providers", skeleton: "chips" },
header: ({ column }) => <DataTableSortHeader column={column} title="Providers" />,
size: 150,
enableSorting: true,
sortingFn: (rowA, rowB) =>
(rowA.original.providers ?? []).join(", ").localeCompare((rowB.original.providers ?? []).join(", ")),
cell: ({ row }) => <ProviderChips providers={row.original.providers ?? []} />,
},
{
id: "mode",
accessorKey: "mode",
meta: { title: "Mode" },
header: ({ column }) => <DataTableSortHeader column={column} title="Mode" />,
size: 110,
enableSorting: true,
sortingFn: "alphanumeric",
cell: ({ row }) => (
<span className="flex items-center gap-2 text-sm">
<span>{getModeIcon(row.original.mode || "")}</span>
<span>{row.original.mode || "Chat"}</span>
</span>
),
},
{
id: "max_input_tokens",
accessorKey: "max_input_tokens",
meta: { title: "Max Input", numeric: true },
header: ({ column }) => <DataTableSortHeader column={column} title="Max Input" />,
size: 100,
enableSorting: true,
cell: ({ row }) => <span className="text-sm">{formatTokens(row.original.max_input_tokens)}</span>,
},
{
id: "max_output_tokens",
accessorKey: "max_output_tokens",
meta: { title: "Max Output", numeric: true },
header: ({ column }) => <DataTableSortHeader column={column} title="Max Output" />,
size: 100,
enableSorting: true,
cell: ({ row }) => <span className="text-sm">{formatTokens(row.original.max_output_tokens)}</span>,
},
{
id: "input_cost_per_token",
accessorKey: "input_cost_per_token",
meta: { title: "Input $/1M", numeric: true },
header: ({ column }) => <DataTableSortHeader column={column} title="Input $/1M" />,
size: 110,
enableSorting: true,
cell: ({ row }) => (
<span className="text-sm">
{row.original.input_cost_per_token ? formatCost(row.original.input_cost_per_token) : "Free"}
</span>
),
},
{
id: "output_cost_per_token",
accessorKey: "output_cost_per_token",
meta: { title: "Output $/1M", numeric: true },
header: ({ column }) => <DataTableSortHeader column={column} title="Output $/1M" />,
size: 110,
enableSorting: true,
cell: ({ row }) => (
<span className="text-sm">
{row.original.output_cost_per_token ? formatCost(row.original.output_cost_per_token) : "Free"}
</span>
),
},
{
id: "features",
meta: { title: "Features", skeleton: "chips" },
header: "Features",
size: 140,
enableSorting: false,
cell: ({ row }) => {
const features = Object.entries(row.original)
.filter(([key, value]) => key.startsWith("supports_") && value === true)
.map(([key]) => formatCapabilityName(key));
return <OverflowChips items={features} />;
},
},
{
id: "health_status",
accessorKey: "health_status",
meta: { title: "Health Status", skeleton: "badge" },
header: ({ column }) => <DataTableSortHeader column={column} title="Health Status" />,
size: 130,
enableSorting: true,
cell: ({ row }) => {
const model = row.original;
const responseTimeLabel = model.health_response_time
? `Response Time: ${Number(model.health_response_time).toFixed(2)}ms`
: "N/A";
const lastCheckedLabel = model.health_checked_at
? `Last Checked: ${new Date(model.health_checked_at).toLocaleString()}`
: "N/A";
return (
<CellTooltip
content={
<>
<div>{responseTimeLabel}</div>
<div>{lastCheckedLabel}</div>
</>
}
trigger={
<span className="capitalize">
<StatusBadge
tone={HEALTH_TONES[model.health_status ?? ""] || "neutral"}
label={model.health_status ?? "Unknown"}
/>
</span>
}
export const getPublicModelHubColumns = ({ onModelClick }: PublicModelHubColumnsDeps): ColumnDef<ModelGroupInfo>[] => {
const columns: ColumnDef<ModelGroupInfo>[] = [
{
id: "model_group",
accessorKey: "model_group",
meta: { title: "Model Name" },
header: ({ column }) => <DataTableSortHeader column={column} title="Model Name" />,
size: 200,
sortingFn: "alphanumeric",
cell: ({ row }) => (
<IdentityCell
title={row.original.model_group}
titleClassName="font-mono text-xs font-normal"
className="max-w-72"
onClick={() => onModelClick(row.original)}
/>
);
),
},
},
{
id: "rpm",
accessorKey: "rpm",
meta: { title: "Limits" },
header: ({ column }) => <DataTableSortHeader column={column} title="Limits" />,
size: 150,
enableSorting: true,
cell: ({ row }) => (
<span className="text-xs text-muted-foreground">{formatLimits(row.original.rpm, row.original.tpm)}</span>
),
},
];
{
id: "providers",
accessorKey: "providers",
meta: { title: "Providers", skeleton: "chips" },
header: ({ column }) => <DataTableSortHeader column={column} title="Providers" />,
size: 150,
sortingFn: (rowA, rowB) =>
(rowA.original.providers ?? []).join(", ").localeCompare((rowB.original.providers ?? []).join(", ")),
cell: ({ row }) => <ProviderChips providers={row.original.providers ?? []} />,
},
{
id: "mode",
accessorKey: "mode",
meta: { title: "Mode" },
header: ({ column }) => <DataTableSortHeader column={column} title="Mode" />,
size: 110,
sortingFn: "alphanumeric",
cell: ({ row }) => (
<span className="flex items-center gap-2 text-sm">
<span>{getModeIcon(row.original.mode || "")}</span>
<span>{row.original.mode || "Chat"}</span>
</span>
),
},
{
id: "max_input_tokens",
accessorKey: "max_input_tokens",
meta: { title: "Max Input", numeric: true },
header: ({ column }) => <DataTableSortHeader column={column} title="Max Input" />,
size: 100,
cell: ({ row }) => <span className="text-sm">{formatTokens(row.original.max_input_tokens)}</span>,
},
{
id: "max_output_tokens",
accessorKey: "max_output_tokens",
meta: { title: "Max Output", numeric: true },
header: ({ column }) => <DataTableSortHeader column={column} title="Max Output" />,
size: 100,
cell: ({ row }) => <span className="text-sm">{formatTokens(row.original.max_output_tokens)}</span>,
},
{
id: "input_cost_per_token",
accessorKey: "input_cost_per_token",
meta: { title: "Input $/1M", numeric: true },
header: ({ column }) => <DataTableSortHeader column={column} title="Input $/1M" />,
size: 110,
cell: ({ row }) => (
<span className="text-sm">
{row.original.input_cost_per_token ? formatCost(row.original.input_cost_per_token) : "Free"}
</span>
),
},
{
id: "output_cost_per_token",
accessorKey: "output_cost_per_token",
meta: { title: "Output $/1M", numeric: true },
header: ({ column }) => <DataTableSortHeader column={column} title="Output $/1M" />,
size: 110,
cell: ({ row }) => (
<span className="text-sm">
{row.original.output_cost_per_token ? formatCost(row.original.output_cost_per_token) : "Free"}
</span>
),
},
{
id: "features",
meta: { title: "Features", skeleton: "chips" },
header: "Features",
size: 140,
cell: ({ row }) => {
const features = Object.entries(row.original)
.filter(([key, value]) => key.startsWith("supports_") && value === true)
.map(([key]) => formatCapabilityName(key));
return <OverflowChips items={features} />;
},
},
{
id: "health_status",
accessorKey: "health_status",
meta: { title: "Health Status", skeleton: "badge" },
header: ({ column }) => <DataTableSortHeader column={column} title="Health Status" />,
size: 130,
cell: ({ row }) => {
const model = row.original;
const responseTimeLabel = model.health_response_time
? `Response Time: ${Number(model.health_response_time).toFixed(2)}ms`
: "N/A";
const lastCheckedLabel = model.health_checked_at
? `Last Checked: ${new Date(model.health_checked_at).toLocaleString()}`
: "N/A";
return (
<CellTooltip
content={
<>
<div>{responseTimeLabel}</div>
<div>{lastCheckedLabel}</div>
</>
}
trigger={
<span className="capitalize">
<StatusBadge
tone={HEALTH_TONES[model.health_status ?? ""] || "neutral"}
label={model.health_status ?? "Unknown"}
/>
</span>
}
/>
);
},
},
{
id: "rpm",
accessorKey: "rpm",
meta: { title: "Limits" },
header: ({ column }) => <DataTableSortHeader column={column} title="Limits" />,
size: 150,
cell: ({ row }) => (
<span className="text-xs text-muted-foreground">{formatLimits(row.original.rpm, row.original.tpm)}</span>
),
},
];
return columns.map((column) => ({
...column,
enableSorting: PUBLIC_MODEL_HUB_SORTABLE_FIELDS.includes(String(column.id)),
}));
};
interface PublicAgentHubColumnsDeps {
onAgentClick: (agent: AgentCard) => void;

85
ui/litellm-dashboard/src/components/publicModelHub/publicModelHubFilters.test.ts Normal file
View file

@ -0,0 +1,85 @@
import type { ColumnFiltersState } from "@tanstack/react-table";
import { describe, expect, it } from "vitest";
import {
FEATURE_FILTER_ID,
MODE_FILTER_ID,
PROVIDER_FILTER_ID,
featureLabel,
readFilterValues,
serializePublicModelHubFilters,
withFilterValue,
} from "./publicModelHubFilters";
describe("serializePublicModelHubFilters", () => {
it("sends each multi-select as the route's comma separated in filter", () => {
const filters: ColumnFiltersState = [
{ id: MODE_FILTER_ID, value: ["chat", "embedding"] },
{ id: PROVIDER_FILTER_ID, value: ["openai", "anthropic"] },
{ id: FEATURE_FILTER_ID, value: ["vision"] },
];
expect(serializePublicModelHubFilters(filters)).toEqual({
"filter[mode][in]": "chat,embedding",
"filter[providers][in]": "openai,anthropic",
"filter[features][in]": "vision",
});
});
it("omits blank filters rather than sending parameters the route rejects", () => {
const filters: ColumnFiltersState = [
{ id: MODE_FILTER_ID, value: [] },
{ id: PROVIDER_FILTER_ID, value: [] },
];
expect(serializePublicModelHubFilters(filters)).toEqual({});
});
it("ignores filter ids the route does not declare", () => {
expect(serializePublicModelHubFilters([{ id: "health_status", value: ["healthy"] }])).toEqual({});
});
});
describe("readFilterValues", () => {
it("reads back the values of the filter it names", () => {
const filters: ColumnFiltersState = [
{ id: MODE_FILTER_ID, value: ["chat"] },
{ id: FEATURE_FILTER_ID, value: ["vision", "reasoning"] },
];
expect(readFilterValues(filters, FEATURE_FILTER_ID)).toEqual(["vision", "reasoning"]);
expect(readFilterValues(filters, PROVIDER_FILTER_ID)).toEqual([]);
});
});
describe("withFilterValue", () => {
it("adds a filter that is not set yet", () => {
expect(withFilterValue([], PROVIDER_FILTER_ID, ["openai"])).toEqual([
{ id: PROVIDER_FILTER_ID, value: ["openai"] },
]);
});
it("replaces a filter instead of stacking a second one", () => {
const filters: ColumnFiltersState = [{ id: PROVIDER_FILTER_ID, value: ["openai"] }];
expect(withFilterValue(filters, PROVIDER_FILTER_ID, ["anthropic"])).toEqual([
{ id: PROVIDER_FILTER_ID, value: ["anthropic"] },
]);
});
it("drops a cleared filter and leaves the others alone", () => {
const filters: ColumnFiltersState = [
{ id: MODE_FILTER_ID, value: ["chat"] },
{ id: PROVIDER_FILTER_ID, value: ["openai"] },
];
expect(withFilterValue(filters, PROVIDER_FILTER_ID, [])).toEqual([{ id: MODE_FILTER_ID, value: ["chat"] }]);
});
});
describe("featureLabel", () => {
it("renders a route feature the way the hub has always labelled it", () => {
expect(featureLabel("vision")).toBe("Vision");
expect(featureLabel("parallel_function_calling")).toBe("Parallel Function Calling");
});
});

60
ui/litellm-dashboard/src/components/publicModelHub/publicModelHubFilters.ts Normal file
View file

@ -0,0 +1,60 @@
import type { ColumnFilter, ColumnFiltersState } from "@tanstack/react-table";
export const MODE_FILTER_ID = "mode";
export const PROVIDER_FILTER_ID = "providers";
export const FEATURE_FILTER_ID = "features";
export const PUBLIC_MODEL_HUB_SORTABLE_FIELDS: readonly string[] = [
"model_group",
"mode",
"providers",
"max_input_tokens",
"max_output_tokens",
"input_cost_per_token",
"output_cost_per_token",
"rpm",
"tpm",
];
type QueryEntry = readonly [string, string];
type FilterValue = string | string[];
const entries = (key: string, value: string): QueryEntry[] => (value === "" ? [] : [[key, value]]);
const asStringArray = (value: unknown): string[] =>
Array.isArray(value) ? value.filter((item): item is string => typeof item === "string") : [];
const inFilter = (field: string, value: unknown): QueryEntry[] =>
entries(`filter[${field}][in]`, asStringArray(value).join(","));
const filterParams = (filter: ColumnFilter): QueryEntry[] => {
switch (filter.id) {
case MODE_FILTER_ID:
case PROVIDER_FILTER_ID:
case FEATURE_FILTER_ID:
return inFilter(filter.id, filter.value);
default:
return [];
}
};
export const serializePublicModelHubFilters = (filters: ColumnFiltersState): Readonly<Record<string, string>> =>
Object.fromEntries(filters.flatMap(filterParams));
export const readFilterValues = (filters: ColumnFiltersState, id: string): string[] =>
asStringArray(filters.find((filter) => filter.id === id)?.value);
const isEmpty = (value: FilterValue): boolean => (Array.isArray(value) ? value.length === 0 : value.trim() === "");
export const withFilterValue = (filters: ColumnFiltersState, id: string, value: FilterValue): ColumnFiltersState => {
const others = filters.filter((filter) => filter.id !== id);
return isEmpty(value) ? others : [...others, { id, value }];
};
/** `supports_vision` reaches the route as `vision`; the hub has always shown it as "Vision". */
export const featureLabel = (feature: string): string =>
feature
.split("_")
.map((word) => word.charAt(0).toUpperCase() + word.slice(1))
.join(" ");

47
ui/litellm-dashboard/src/components/publicModelHub/usePublicModelHubFacets.ts Normal file
View file

@ -0,0 +1,47 @@
"use client";
import { useQueries } from "@tanstack/react-query";
import { apiClient } from "@/components/networking";
import type { components } from "@/lib/http/schema";
import { PUBLIC_MODEL_HUB_PATH } from "./usePublicModelHubList";
type FacetResponse = components["schemas"]["FacetListResponse"];
export const MODEL_HUB_FACETS = ["providers", "modes", "features"] as const;
export type ModelHubFacet = (typeof MODEL_HUB_FACETS)[number];
/** The route caps a page at 100, which is far above the distinct providers, modes or features any proxy publishes. */
const FACET_PAGE_SIZE = 100;
export interface PublicModelHubFacets {
providers: string[];
modes: string[];
features: string[];
}
const fetchFacet = (facet: ModelHubFacet, signal: AbortSignal): Promise<FacetResponse> =>
apiClient.get<FacetResponse>(`${PUBLIC_MODEL_HUB_PATH}/${facet}`, {
query: { page_size: FACET_PAGE_SIZE },
signal,
});
/**
* The values each filter dropdown offers, read from the route rather than derived from a
* page of rows, which can only ever show the values that page happens to contain.
*/
export const usePublicModelHubFacets = (enabled: boolean): PublicModelHubFacets => {
const results = useQueries({
queries: MODEL_HUB_FACETS.map((facet) => ({
queryKey: ["publicModelHub", "facet", facet],
queryFn: ({ signal }: { signal: AbortSignal }) => fetchFacet(facet, signal),
enabled,
staleTime: Infinity,
})),
});
const [providers, modes, features] = results.map((result) => result.data?.data ?? []);
return { providers, modes, features };
};

83
ui/litellm-dashboard/src/components/publicModelHub/usePublicModelHubList.ts Normal file
View file

@ -0,0 +1,83 @@
"use client";
import type { SortingState } from "@tanstack/react-table";
import { useCallback } from "react";
import {
useResourceList,
type ResourceListPage,
type ResourceListQuery,
type ResourceListResult,
} from "@/app/(dashboard)/hooks/common/useResourceList";
import { apiClient } from "@/components/networking";
import type { ModelGroupInfo } from "@/components/PublicModelHubTableColumns";
import {
FEATURE_FILTER_ID,
MODE_FILTER_ID,
PROVIDER_FILTER_ID,
readFilterValues,
serializePublicModelHubFilters,
withFilterValue,
} from "./publicModelHubFilters";
export const PUBLIC_MODEL_HUB_PATH = "/public/v1/model_hub";
export const PUBLIC_MODEL_HUB_PAGE_SIZE = 50;
const QUERY_KEY = ["publicModelHub", "list"] as const;
const DEFAULT_SORTING: SortingState = [{ id: "model_group", desc: false }];
export interface PublicModelHubListResult extends ResourceListResult<ModelGroupInfo> {
providerValues: string[];
onProvidersChange: (values: string[]) => void;
modeValues: string[];
onModesChange: (values: string[]) => void;
featureValues: string[];
onFeaturesChange: (values: string[]) => void;
hasActiveQuery: boolean;
}
const fetchPage = async (query: ResourceListQuery, signal: AbortSignal): Promise<ResourceListPage<ModelGroupInfo>> => {
try {
return await apiClient.get<ResourceListPage<ModelGroupInfo>>(PUBLIC_MODEL_HUB_PATH, { query, signal });
} catch (error) {
if (!signal.aborted) {
console.error("There was an error fetching the public model data", error);
}
throw error;
}
};
export const usePublicModelHubList = (enabled: boolean): PublicModelHubListResult => {
const listOptions = {
queryKey: QUERY_KEY,
fetchPage,
serializeFilters: serializePublicModelHubFilters,
defaultSorting: DEFAULT_SORTING,
defaultPageSize: PUBLIC_MODEL_HUB_PAGE_SIZE,
enabled,
};
const list = useResourceList<ModelGroupInfo>(listOptions);
const { onColumnFiltersChange } = list;
const setFilter = useCallback(
(id: string, values: string[]) => onColumnFiltersChange((previous) => withFilterValue(previous, id, values)),
[onColumnFiltersChange],
);
const onProvidersChange = useCallback((values: string[]) => setFilter(PROVIDER_FILTER_ID, values), [setFilter]);
const onModesChange = useCallback((values: string[]) => setFilter(MODE_FILTER_ID, values), [setFilter]);
const onFeaturesChange = useCallback((values: string[]) => setFilter(FEATURE_FILTER_ID, values), [setFilter]);
return {
...list,
providerValues: readFilterValues(list.columnFilters, PROVIDER_FILTER_ID),
onProvidersChange,
modeValues: readFilterValues(list.columnFilters, MODE_FILTER_ID),
onModesChange,
featureValues: readFilterValues(list.columnFilters, FEATURE_FILTER_ID),
onFeaturesChange,
hasActiveQuery: list.searchValue.trim() !== "" || list.columnFilters.length > 0,
};
};

323
ui/litellm-dashboard/src/components/public_model_hub.test.tsx
View file

@ -1,8 +1,12 @@
import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
import { describe, it, expect, vi, beforeAll, beforeEach } from "vitest";
import { render, screen, waitFor, within, fireEvent } from "@testing-library/react";
import userEvent from "@testing-library/user-event";
import { flexRender, getCoreRowModel, useReactTable } from "@tanstack/react-table";
import PublicModelHub from "./public_model_hub";
import { getPublicMCPHubColumns, MCPServerData } from "./PublicModelHubTableColumns";
import { getPublicMCPHubColumns, MCPServerData, ModelGroupInfo } from "./PublicModelHubTableColumns";
const { apiGetMock } = vi.hoisted(() => ({ apiGetMock: vi.fn() }));
vi.mock("next/navigation", () => ({
useRouter: vi.fn(() => ({
@ -16,6 +20,7 @@ vi.mock("./networking", async (importOriginal) => {
const actual = await importOriginal<typeof import("./networking")>();
return {
...actual,
apiClient: { ...actual.apiClient, get: apiGetMock },
modelHubPublicModelsCall: vi.fn().mockResolvedValue([]),
getPublicModelHubInfo: vi.fn().mockResolvedValue({
docs_title: "LiteLLM Gateway",
@ -34,6 +39,68 @@ vi.mock("./navbar", () => ({
default: vi.fn(() => <div data-testid="navbar">Navbar Component</div>),
}));
const MODEL_HUB_PATH = "/public/v1/model_hub";
const FACET_VALUES: Record<string, string[]> = {
[`${MODEL_HUB_PATH}/providers`]: ["anthropic", "openai"],
[`${MODEL_HUB_PATH}/modes`]: ["chat", "embedding"],
[`${MODEL_HUB_PATH}/features`]: ["function_calling", "vision"],
};
const MODEL_DEFAULTS = {
providers: ["openai"],
mode: "chat",
supports_function_calling: false,
supports_vision: false,
supports_parallel_function_calling: false,
};
const model = (overrides: Partial<ModelGroupInfo> & { model_group: string }): ModelGroupInfo => ({
...MODEL_DEFAULTS,
...overrides,
});
const DEFAULT_MODELS = [model({ model_group: "gpt-4" }), model({ model_group: "claude-3", providers: ["anthropic"] })];
const respondWith = (rows: ModelGroupInfo[], totalCount: number = rows.length, pageSize: number = 50) =>
apiGetMock.mockImplementation((path: string) => {
const facet = FACET_VALUES[path];
if (facet) {
return Promise.resolve({
data: facet,
meta: { page: 1, page_size: 100, has_more: false },
links: { self: path, prev: null, next: null },
});
}
return Promise.resolve({
data: rows,
meta: {
total_count: totalCount,
page: 1,
page_size: pageSize,
total_pages: Math.max(Math.ceil(totalCount / pageSize), 1),
},
links: { self: MODEL_HUB_PATH, first: MODEL_HUB_PATH, prev: null, next: null, last: MODEL_HUB_PATH },
});
});
type QueryRecord = Record<string, string | number>;
const modelCalls = () => apiGetMock.mock.calls.filter((call) => call[0] === MODEL_HUB_PATH);
const facetPaths = (): string[] =>
apiGetMock.mock.calls.map((call) => String(call[0])).filter((path) => path.startsWith(`${MODEL_HUB_PATH}/`));
const modelQueries = (): QueryRecord[] => modelCalls().map((call) => (call[1] as { query: QueryRecord }).query);
const lastModelQuery = (): QueryRecord => modelQueries()[modelQueries().length - 1];
const renderHub = () => {
const client = new QueryClient({ defaultOptions: { queries: { retry: false, gcTime: 0 } } });
return render(
<QueryClientProvider client={client}>
<PublicModelHub />
</QueryClientProvider>,
);
};
beforeAll(() => {
Object.defineProperty(window, "matchMedia", {
writable: true,
@ -51,6 +118,8 @@ beforeAll(() => {
});
beforeEach(() => {
vi.clearAllMocks();
respondWith(DEFAULT_MODELS);
Storage.prototype.getItem = vi.fn(() => "false");
Storage.prototype.setItem = vi.fn();
Object.defineProperty(window, "location", {
@ -64,58 +133,215 @@ beforeEach(() => {
describe("PublicModelHub", () => {
it("renders", () => {
const { container } = render(<PublicModelHub />);
const { container } = renderHub();
expect(container).toBeInTheDocument();
});
it("loads the first page of models from the paginated public endpoint", async () => {
renderHub();
expect(await screen.findByText("gpt-4")).toBeInTheDocument();
expect(modelCalls()[0][0]).toBe(MODEL_HUB_PATH);
expect(modelQueries()[0]).toEqual({ page: 1, page_size: 50, sort: "model_group" });
});
it("waits for the resolved proxy base url before asking for a page", async () => {
const networkingModule = await import("./networking");
let publishConfig: () => void = () => {};
vi.mocked(networkingModule.getUiConfig).mockReturnValueOnce(
new Promise((resolve) => {
publishConfig = () => resolve({} as Awaited<ReturnType<typeof networkingModule.getUiConfig>>);
}),
);
renderHub();
await new Promise((resolve) => setTimeout(resolve, 50));
expect(modelCalls()).toHaveLength(0);
publishConfig();
await waitFor(() => expect(modelCalls().length).toBeGreaterThan(0));
});
it("stops calling the unpaginated public model hub route", async () => {
const networkingModule = await import("./networking");
renderHub();
await waitFor(() => expect(apiGetMock).toHaveBeenCalled());
expect(networkingModule.modelHubPublicModelsCall).not.toHaveBeenCalled();
});
it("counts the whole catalogue from the response meta, not the rows on screen", async () => {
respondWith(DEFAULT_MODELS, 300);
renderHub();
await screen.findByText("gpt-4");
expect(screen.getByTestId("pagination-range")).toHaveTextContent("of 300");
expect(screen.getByTestId("pagination-page")).toHaveTextContent("Page 1 of 6");
});
it("asks the server for the next page", async () => {
const user = userEvent.setup();
respondWith(DEFAULT_MODELS, 300);
renderHub();
await screen.findByText("gpt-4");
await user.click(screen.getByTestId("pagination-next"));
await waitFor(() => expect(lastModelQuery().page).toBe(2));
expect(lastModelQuery().page_size).toBe(50);
});
it("asks the server for a different page size", async () => {
const user = userEvent.setup();
respondWith(DEFAULT_MODELS, 300);
renderHub();
await screen.findByText("gpt-4");
await user.click(screen.getByTestId("pagination-page-size"));
await user.click(await screen.findByRole("option", { name: "25" }));
await waitFor(() => expect(lastModelQuery().page_size).toBe(25));
});
it("asks the server to sort, in the sort form the endpoint accepts", async () => {
const user = userEvent.setup();
renderHub();
await screen.findByText("gpt-4");
await user.click(screen.getByTestId("sort-header-model_group"));
await waitFor(() => expect(lastModelQuery().sort).toBe("-model_group"));
await user.click(screen.getByTestId("sort-header-input_cost_per_token"));
await waitFor(() => expect(lastModelQuery().sort).toBe("-input_cost_per_token"));
});
it("renders the page in the order the server sent it, without re-sorting locally", async () => {
const user = userEvent.setup();
respondWith([model({ model_group: "alpha-model" }), model({ model_group: "zeta-model" })], 300);
renderHub();
await screen.findByText("alpha-model");
await user.click(screen.getByTestId("sort-header-model_group"));
await waitFor(() => expect(lastModelQuery().sort).toBe("-model_group"));
const rendered = screen.getAllByText(/-model$/).map((cell) => cell.textContent);
expect(rendered).toEqual(["alpha-model", "zeta-model"]);
});
it("offers sorting on exactly the fields the endpoint accepts", async () => {
renderHub();
await screen.findByText("gpt-4");
const sortable = screen
.getAllByTestId(/^sort-header-/)
.map((header) => header.getAttribute("data-testid")?.replace("sort-header-", ""));
expect(sortable.sort()).toEqual([
"input_cost_per_token",
"max_input_tokens",
"max_output_tokens",
"mode",
"model_group",
"output_cost_per_token",
"providers",
"rpm",
]);
expect(screen.getByText("Health Status")).toBeInTheDocument();
expect(screen.queryByTestId("sort-header-health_status")).not.toBeInTheDocument();
});
it("searches on the server and returns to the first page", async () => {
const user = userEvent.setup();
respondWith(DEFAULT_MODELS, 300);
renderHub();
await screen.findByText("gpt-4");
await user.click(screen.getByTestId("pagination-next"));
await waitFor(() => expect(lastModelQuery().page).toBe(2));
await user.type(screen.getByPlaceholderText("Search model names..."), "claude");
await waitFor(() => expect(lastModelQuery().q).toBe("claude"));
expect(lastModelQuery().page).toBe(1);
});
it("filters by mode with the endpoint's in operator", async () => {
const user = userEvent.setup();
renderHub();
await screen.findByText("gpt-4");
await user.click(screen.getByPlaceholderText("Select modes"));
await user.click(await screen.findByRole("option", { name: "embedding" }));
await waitFor(() => expect(lastModelQuery()["filter[mode][in]"]).toBe("embedding"));
});
it("filters by several providers at once, and returns to the first page", async () => {
const user = userEvent.setup();
respondWith(DEFAULT_MODELS, 300);
renderHub();
await screen.findByText("gpt-4");
await user.click(screen.getByTestId("pagination-next"));
await waitFor(() => expect(lastModelQuery().page).toBe(2));
await user.click(screen.getByPlaceholderText("Select providers"));
await user.click(await screen.findByRole("option", { name: /anthropic/i }));
await waitFor(() => expect(lastModelQuery()["filter[providers][in]"]).toBe("anthropic"));
expect(lastModelQuery().page).toBe(1);
await user.click(await screen.findByRole("option", { name: /openai/i }));
await waitFor(() => expect(lastModelQuery()["filter[providers][in]"]).toBe("anthropic,openai"));
});
it("filters by feature, which the table could not do while it paged", async () => {
const user = userEvent.setup();
renderHub();
await screen.findByText("gpt-4");
await user.click(screen.getByPlaceholderText("Select features"));
await user.click(await screen.findByRole("option", { name: "Vision" }));
await waitFor(() => expect(lastModelQuery()["filter[features][in]"]).toBe("vision"));
});
it("offers the filter values the route reports, not the ones on the page", async () => {
respondWith([model({ model_group: "gpt-4" })], 1);
renderHub();
await screen.findByText("gpt-4");
await waitFor(() => expect(facetPaths()).toContain(`${MODEL_HUB_PATH}/providers`));
expect(facetPaths()).toEqual(expect.arrayContaining([`${MODEL_HUB_PATH}/modes`, `${MODEL_HUB_PATH}/features`]));
});
it("displays health status correctly for models with health check information", async () => {
const mockModelsWithHealthChecks = [
respondWith([
{
...MODEL_DEFAULTS,
model_group: "gpt-4",
providers: ["openai"],
mode: "chat",
health_status: "healthy",
health_response_time: 150.5,
health_checked_at: "2024-01-15T10:30:00Z",
supports_function_calling: true,
supports_vision: false,
supports_parallel_function_calling: false,
},
{
...MODEL_DEFAULTS,
model_group: "claude-3",
providers: ["anthropic"],
mode: "chat",
health_status: "unhealthy",
health_response_time: 5000.0,
health_checked_at: "2024-01-15T10:25:00Z",
supports_function_calling: true,
supports_vision: false,
supports_parallel_function_calling: false,
},
{
model_group: "gpt-3.5-turbo",
providers: ["openai"],
mode: "chat",
health_status: undefined,
health_response_time: undefined,
health_checked_at: undefined,
supports_function_calling: false,
supports_vision: false,
supports_parallel_function_calling: false,
},
];
model({ model_group: "gpt-3.5-turbo" }),
]);
const networkingModule = await import("./networking");
vi.mocked(networkingModule.modelHubPublicModelsCall).mockResolvedValue(mockModelsWithHealthChecks);
renderHub();
render(<PublicModelHub />);
// Wait for the component to load and render the table
await waitFor(() => {
expect(screen.getByText("gpt-4")).toBeInTheDocument();
});
// Check the health status badge in each model's row
await waitFor(() => {
const gpt4Row = screen.getByText("gpt-4").closest("tr");
expect(gpt4Row).toBeInTheDocument();
@ -134,19 +360,13 @@ describe("PublicModelHub", () => {
expect(within(gpt35Row as HTMLElement).getByText("Unknown")).toBeInTheDocument();
});
});
it("shows no models when the search has no matches (LIT-5230 regression)", async () => {
const networkingModule = await import("./networking");
vi.mocked(networkingModule.modelHubPublicModelsCall).mockResolvedValue([
{ model_group: "gpt-4", providers: ["openai"], mode: "chat" },
{ model_group: "claude-3", providers: ["anthropic"], mode: "chat" },
]);
render(<PublicModelHub />);
it("shows no models when the search has no matches (LIT-5230 regression)", async () => {
renderHub();
expect(await screen.findByText("gpt-4")).toBeInTheDocument();
fireEvent.change(screen.getByPlaceholderText("Search model names... (smart search enabled)"), {
target: { value: "zzzz" },
});
respondWith([], 0);
fireEvent.change(screen.getByPlaceholderText("Search model names..."), { target: { value: "zzzz" } });
await waitFor(() => {
expect(screen.queryByText("gpt-4")).not.toBeInTheDocument();
@ -155,18 +375,23 @@ describe("PublicModelHub", () => {
});
});
it("handles non-array response gracefully (regression test for e.filter crash)", async () => {
const networkingModule = await import("./networking");
// Mock the API to return an object (like an error response) instead of an array
vi.mocked(networkingModule.modelHubPublicModelsCall).mockResolvedValue({
detail: "No models configured",
} as any);
it("reports the proxy as unavailable when the model page fails to load", async () => {
apiGetMock.mockRejectedValue(new Error("boom"));
render(<PublicModelHub />);
renderHub();
expect(await screen.findByText(/Service unavailable/)).toBeInTheDocument();
});
it("keeps the page usable when the response carries no rows", async () => {
respondWith([], 0);
renderHub();
await waitFor(() => {
expect(screen.getByTestId("navbar")).toBeInTheDocument();
expect(screen.getByText("Model Hub")).toBeInTheDocument();
expect(screen.getByText("No models available")).toBeInTheDocument();
});
});
});
@ -237,7 +462,7 @@ describe("public hub MCP details modal", () => {
const networkingModule = await import("./networking");
vi.mocked(networkingModule.mcpHubPublicServersCall).mockResolvedValue([mockMcpServer]);
render(<PublicModelHub />);
renderHub();
fireEvent.click(await screen.findByRole("tab", { name: /MCP Hub/i }));
fireEvent.click(await screen.findByRole("button", { name: "exa_test" }));
@ -252,7 +477,7 @@ describe("public hub MCP details modal", () => {
const networkingModule = await import("./networking");
vi.mocked(networkingModule.mcpHubPublicServersCall).mockResolvedValue([mockMcpServer]);
render(<PublicModelHub />);
renderHub();
fireEvent.click(await screen.findByRole("tab", { name: /MCP Hub/i }));
fireEvent.click(await screen.findByRole("button", { name: "exa_test" }));

176
ui/litellm-dashboard/src/components/public_model_hub.tsx
View file

@ -21,6 +21,9 @@ import { Dialog, DialogContent, DialogHeader, DialogTitle } from "@/components/u
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip";
import { MultiSelect } from "./shared/MultiSelect";
import { featureLabel } from "./publicModelHub/publicModelHubFilters";
import { usePublicModelHubFacets } from "./publicModelHub/usePublicModelHubFacets";
import { usePublicModelHubList } from "./publicModelHub/usePublicModelHubList";
import { DataTable } from "./shared/DataTable";
import { toast } from "@/lib/toast";
import Navbar from "./navbar";
@ -31,7 +34,6 @@ import {
getPublicModelHubInfo,
getUiConfig,
mcpHubPublicServersCall,
modelHubPublicModelsCall,
} from "./networking";
import { Plugin } from "./claude_code_plugins/types";
import SkillHubDashboard from "./AIHub/SkillHubDashboard";
@ -68,25 +70,19 @@ function PublicHubEmptyState({ title, body }: { title: string; body: string }) {
const PublicModelHub: React.FC<PublicModelHubProps> = ({ accessToken, isEmbedded = false }) => {
const anchor = useComboboxAnchor();
const [modelHubData, setModelHubData] = useState<ModelGroupInfo[] | null>(null);
const [proxyConfigured, setProxyConfigured] = useState<boolean>(false);
const [agentHubData, setAgentHubData] = useState<AgentCard[] | null>(null);
const [mcpHubData, setMcpHubData] = useState<MCPServerData[] | null>(null);
const [pageTitle, setPageTitle] = useState<string>("LiteLLM Gateway");
const [customDocsDescription, setCustomDocsDescription] = useState<string | null>(null);
const [litellmVersion, setLitellmVersion] = useState<string>("");
const [usefulLinks, setUsefulLinks] = useState<Record<string, string | { url: string; index: number }>>({});
const [loading, setLoading] = useState<boolean>(true);
const [agentLoading, setAgentLoading] = useState<boolean>(true);
const [mcpLoading, setMcpLoading] = useState<boolean>(true);
const [searchTerm, setSearchTerm] = useState<string>("");
const [agentSearchTerm, setAgentSearchTerm] = useState<string>("");
const [mcpSearchTerm, setMcpSearchTerm] = useState<string>("");
const [selectedProviders, setSelectedProviders] = useState<string[]>([]);
const [selectedModes, setSelectedModes] = useState<string[]>([]);
const [selectedFeatures, setSelectedFeatures] = useState<string[]>([]);
const [selectedAgentSkills, setSelectedAgentSkills] = useState<string[]>([]);
const [selectedMcpTransports, setSelectedMcpTransports] = useState<string[]>([]);
const [serviceStatus, setServiceStatus] = useState<string>("I'm alive! ✓");
const [isModalVisible, setIsModalVisible] = useState(false);
const [isAgentModalVisible, setIsAgentModalVisible] = useState(false);
const [isMcpModalVisible, setIsMcpModalVisible] = useState(false);
@ -106,19 +102,7 @@ const PublicModelHub: React.FC<PublicModelHubProps> = ({ accessToken, isEmbedded
console.error("Failed to get UI config:", error);
// Continue anyway - might work with default proxyBaseUrl
}
const fetchPublicData = async () => {
try {
setLoading(true);
const _modelHubData = await modelHubPublicModelsCall();
setModelHubData(Array.isArray(_modelHubData) ? _modelHubData : []);
} catch (error) {
console.error("There was an error fetching the public model data", error);
setServiceStatus("Service unavailable");
} finally {
setLoading(false);
}
};
setProxyConfigured(true);
const fetchAgentData = async () => {
try {
@ -166,7 +150,6 @@ const PublicModelHub: React.FC<PublicModelHubProps> = ({ accessToken, isEmbedded
fetchPublicModelHubInfo();
fetchPublicData();
fetchAgentData();
fetchMcpData();
fetchSkillData();
@ -175,47 +158,6 @@ const PublicModelHub: React.FC<PublicModelHubProps> = ({ accessToken, isEmbedded
initializeAndFetch();
}, []);
// Clear filters when filter values change to avoid confusion
useEffect(() => {
// This would clear selections if we had any selection functionality
// For now, it's just for consistency with the original component
}, [searchTerm, selectedProviders, selectedModes, selectedFeatures]);
const getUniqueProviders = (data: ModelGroupInfo[]) => {
const providers = new Set<string>();
data.forEach((model) => {
(model.providers ?? []).forEach((provider) => providers.add(provider));
});
return Array.from(providers);
};
const getUniqueModes = (data: ModelGroupInfo[]) => {
const modes = new Set<string>();
data.forEach((model) => {
if (model.mode) modes.add(model.mode);
});
return Array.from(modes);
};
const getUniqueFeatures = (data: ModelGroupInfo[]) => {
const features = new Set<string>();
data.forEach((model) => {
// Find all properties that start with 'supports_' and are true
Object.entries(model)
.filter(([key, value]) => key.startsWith("supports_") && value === true)
.forEach(([key]) => {
// Format the feature name (remove 'supports_' prefix and convert to title case)
const featureName = key
.replace(/^supports_/, "")
.split("_")
.map((word) => word.charAt(0).toUpperCase() + word.slice(1))
.join(" ");
features.add(featureName);
});
});
return Array.from(features).sort();
};
const getUniqueAgentSkills = (data: AgentCard[]) => {
const skills = new Set<string>();
data.forEach((agent) => {
@ -234,39 +176,6 @@ const PublicModelHub: React.FC<PublicModelHubProps> = ({ accessToken, isEmbedded
return Array.from(transports).sort();
};
const filteredData = useMemo(() => {
if (!modelHubData || !Array.isArray(modelHubData)) return [];
const searchResults = rankBySearchRelevance(
filterBySearchTerm(modelHubData, searchTerm, (model) => [model.model_group]),
searchTerm,
(model) => model.model_group,
);
// Apply other filters
return searchResults.filter((model) => {
const matchesProvider =
selectedProviders.length === 0 || selectedProviders.some((provider) => model.providers.includes(provider));
const matchesMode = selectedModes.length === 0 || selectedModes.includes(model.mode || "");
// Check if model has any of the selected features
const matchesFeature =
selectedFeatures.length === 0 ||
Object.entries(model)
.filter(([key, value]) => key.startsWith("supports_") && value === true)
.some(([key]) => {
const featureName = key
.replace(/^supports_/, "")
.split("_")
.map((word) => word.charAt(0).toUpperCase() + word.slice(1))
.join(" ");
return selectedFeatures.includes(featureName);
});
return matchesProvider && matchesMode && matchesFeature;
});
}, [modelHubData, searchTerm, selectedProviders, selectedModes, selectedFeatures]);
const filteredAgentData = useMemo(() => {
if (!agentHubData || !Array.isArray(agentHubData)) return [];
@ -356,7 +265,14 @@ const PublicModelHub: React.FC<PublicModelHubProps> = ({ accessToken, isEmbedded
return `$${(cost * 1_000_000).toFixed(4)}`;
};
const [modelSorting, setModelSorting] = useState<SortingState>([{ id: "model_group", desc: false }]);
const models = usePublicModelHubList(proxyConfigured);
const modelFacets = usePublicModelHubFacets(proxyConfigured);
const modeOptions = useMemo(() => modelFacets.modes.map((mode) => ({ label: mode, value: mode })), [modelFacets]);
const featureOptions = useMemo(
() => modelFacets.features.map((feature) => ({ label: featureLabel(feature), value: feature })),
[modelFacets],
);
const serviceStatus = models.error ? "Service unavailable" : "I'm alive! ✓";
const [agentSorting, setAgentSorting] = useState<SortingState>([{ id: "name", desc: false }]);
const [mcpSorting, setMcpSorting] = useState<SortingState>([{ id: "server_name", desc: false }]);
@ -367,22 +283,6 @@ const PublicModelHub: React.FC<PublicModelHubProps> = ({ accessToken, isEmbedded
const hasAgents = Array.isArray(agentHubData) && agentHubData.length > 0;
const hasMcpServers = Array.isArray(mcpHubData) && mcpHubData.length > 0;
const providerOptions = useMemo(
() => (Array.isArray(modelHubData) ? getUniqueProviders(modelHubData) : []),
[modelHubData],
);
const modeOptions = useMemo(
() =>
Array.isArray(modelHubData) ? getUniqueModes(modelHubData).map((mode) => ({ label: mode, value: mode })) : [],
[modelHubData],
);
const featureOptions = useMemo(
() =>
Array.isArray(modelHubData)
? getUniqueFeatures(modelHubData).map((feature) => ({ label: feature, value: feature }))
: [],
[modelHubData],
);
const agentSkillOptions = useMemo(
() =>
Array.isArray(agentHubData)
@ -495,9 +395,8 @@ const PublicModelHub: React.FC<PublicModelHubProps> = ({ accessToken, isEmbedded
<Tooltip>
<TooltipTrigger render={<Info className="w-4 h-4 text-muted-foreground cursor-help" />} />
<TooltipContent side="top">
Smart search with relevance ranking - finds models containing your search terms, ranked by
relevance. Try searching &apos;xai grok-4&apos;, &apos;claude-4&apos;, &apos;gpt-4&apos;, or
&apos;sonnet&apos;
Finds every published model whose name contains what you type, across all pages. Try
&apos;grok&apos;, &apos;claude&apos;, &apos;gpt-4&apos;, or &apos;sonnet&apos;
</TooltipContent>
</Tooltip>
</div>
@ -505,9 +404,10 @@ const PublicModelHub: React.FC<PublicModelHubProps> = ({ accessToken, isEmbedded
<SearchIcon className="w-4 h-4 text-muted-foreground absolute left-3 top-1/2 transform -translate-y-1/2" />
<input
type="text"
placeholder="Search model names... (smart search enabled)"
value={searchTerm}
onChange={(e) => setSearchTerm(e.target.value)}
placeholder="Search model names..."
aria-label="Search model names"
value={models.searchValue}
onChange={(e) => models.onSearchChange(e.target.value)}
className="border border-border rounded-lg pl-10 pr-4 py-2 w-full text-sm focus:outline-hidden focus:ring-2 focus:ring-ring focus:border-transparent bg-card"
/>
</div>
@ -516,9 +416,9 @@ const PublicModelHub: React.FC<PublicModelHubProps> = ({ accessToken, isEmbedded
<p className="text-sm font-medium mb-3 text-foreground">Provider:</p>
<Combobox
multiple
items={providerOptions}
value={selectedProviders}
onValueChange={(values: string[]) => setSelectedProviders(values)}
items={modelFacets.providers}
value={models.providerValues}
onValueChange={models.onProvidersChange}
>
<ComboboxChips render={<div ref={anchor} />} className="min-h-8 w-full py-1 text-sm">
<ComboboxValue>
@ -567,8 +467,8 @@ const PublicModelHub: React.FC<PublicModelHubProps> = ({ accessToken, isEmbedded
<p className="text-sm font-medium mb-3 text-foreground">Mode:</p>
<MultiSelect
options={modeOptions}
value={selectedModes}
onValueChange={setSelectedModes}
value={models.modeValues}
onValueChange={models.onModesChange}
placeholder="Select modes"
className="w-full"
/>
@ -577,8 +477,8 @@ const PublicModelHub: React.FC<PublicModelHubProps> = ({ accessToken, isEmbedded
<p className="text-sm font-medium mb-3 text-foreground">Features:</p>
<MultiSelect
options={featureOptions}
value={selectedFeatures}
onValueChange={setSelectedFeatures}
value={models.featureValues}
onValueChange={models.onFeaturesChange}
placeholder="Select features"
className="w-full"
/>
@ -586,19 +486,23 @@ const PublicModelHub: React.FC<PublicModelHubProps> = ({ accessToken, isEmbedded
</div>
<DataTable
data={filteredData}
data={models.rows}
columns={modelColumns}
getRowId={(model, index) => model.model_group || String(index)}
sortingMode="client"
sorting={modelSorting}
onSortingChange={setModelSorting}
isLoading={loading}
sortingMode="server"
sorting={models.sorting}
onSortingChange={models.onSortingChange}
paginationMode="server"
pagination={models.pagination}
onPaginationChange={models.onPaginationChange}
rowCount={models.rowCount}
isLoading={models.isLoading}
loadingMessage="Loading models…"
noDataMessage={
<PublicHubEmptyState
title={modelHubData?.length ? "No matching models" : "No models available"}
title={models.hasActiveQuery ? "No matching models" : "No models available"}
body={
modelHubData?.length
models.hasActiveQuery
? "Adjust the search or filters to see more models."
: "Models made public by the proxy admin will appear here."
}
@ -606,12 +510,6 @@ const PublicModelHub: React.FC<PublicModelHubProps> = ({ accessToken, isEmbedded
}
size="compact"
/>
<div className="mt-8 text-center">
<p className="text-sm text-muted-foreground">
Showing {filteredData.length} of {modelHubData?.length || 0} models
</p>
</div>
</TabsContent>
{/* Agents Tab */}

61
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -12382,6 +12382,36 @@ export interface paths {
patch?: never;
trace?: never;
};
"/public/v1/model_hub/{facet}": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
/**
* Public Model Hub Facet
* @description The distinct providers, modes or features across the published model groups, for the
* Model Hub's filter dropdowns. No authentication.
*
* Carries the same filters and search as the list route, so a dropdown offers exactly
* the values the table can show: asking for providers under `filter[mode][in]=chat`
* lists only the providers that serve a chat model.
*
* Example curl:
* ```
* curl --location --globoff 'http://0.0.0.0:4000/public/v1/model_hub/providers?filter[mode][in]=chat&page_size=50'
* ```
*/
get: operations["public_model_hub_facet_public_v1_model_hub__facet__get"];
put?: never;
post?: never;
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/queue/chat/completions": {
parameters: {
query?: never;
@ -55040,6 +55070,37 @@ export interface operations {
};
};
};
public_model_hub_facet_public_v1_model_hub__facet__get: {
parameters: {
query?: never;
header?: never;
path: {
facet: "providers" | "modes" | "features";
};
cookie?: never;
};
requestBody?: never;
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["FacetListResponse"];
};
};
/** @description Validation Error */
422: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["HTTPValidationError"];
};
};
};
};
async_queue_request_queue_chat_completions_post: {
parameters: {
query?: {