diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 1a807fd39bb..832d941f5b5 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -737,6 +737,9 @@ class LiteLLMRoutes(enum.Enum): "/.well-known/litellm-ui-config", "/public/model_hub", "/public/v1/model_hub", + "/public/v1/model_hub/providers", + "/public/v1/model_hub/modes", + "/public/v1/model_hub/features", "/public/model_hub/info", "/public/agent_hub", "/public/mcp_hub", diff --git a/litellm/proxy/list_api/in_memory.py b/litellm/proxy/list_api/in_memory.py index bada8ea0a35..3f3f173ba1a 100644 --- a/litellm/proxy/list_api/in_memory.py +++ b/litellm/proxy/list_api/in_memory.py @@ -141,3 +141,15 @@ class InMemoryListExecutor(Generic[TRow]): async def find_many(self, plan: QueryPlan) -> Sequence[TRow]: page: Final = _ordered(self._matching(plan.where), plan.order)[plan.skip : plan.skip + plan.take] return await self.enrich_page(tuple(row for _, row in page)) + + async def distinct(self, field: str, where: tuple[Predicate, ...]) -> Sequence[str]: + """A repeated field contributes each of its elements, so a facet over `providers` + lists providers rather than the tuples rows happen to carry.""" + cells: Final = (cells.get(field) for cells, _ in self._matching(where)) + values: Final = ( + value + for cell in cells + for value in (cell if isinstance(cell, tuple) else (cell,)) + if isinstance(value, str) and value + ) + return tuple(sorted(frozenset(values))) diff --git a/litellm/proxy/list_api/list_framework.py b/litellm/proxy/list_api/list_framework.py index 21ee4e6860f..af422b7650a 100644 --- a/litellm/proxy/list_api/list_framework.py +++ b/litellm/proxy/list_api/list_framework.py @@ -28,12 +28,15 @@ from litellm.proxy.list_api.common import ( PROBLEM_TYPE_BASE, ManagementProblem, build_list_links, + build_page_links, escape_like, unknown_query_param_problem, ) from litellm.types.proxy.management_endpoints.management_v1 import ( + FacetListResponse, ListMeta, ListResponse, + PageMeta, ProblemDetail, ) @@ -186,6 +189,13 @@ class ListExecutor(Protocol[TRow_co]): async def find_many(self, plan: QueryPlan) -> Sequence[TRow_co]: ... +class FacetExecutor(Protocol): + """The half of a facet that knows the rows. Separate from `ListExecutor` so a SQL + executor is not forced to implement `distinct` to keep serving entity lists.""" + + async def distinct(self, field: str, where: tuple[Predicate, ...]) -> Sequence[str]: ... + + def order_by_sql(order: tuple[SortKey, ...]) -> str: """`ORDER BY` body for a plan, NULLS LAST in both directions. @@ -515,6 +525,78 @@ def build_query_plan( ) +def _facet_allowed_params(spec: ListSpec[TRow, TOut]) -> tuple[str, ...]: + """A facet's values are always ascending, so `sort` is not one of its parameters.""" + return tuple(name for name in _allowed_params(spec) if name != SORT_PARAM) + + +def _facet_where( + spec: ListSpec[TRow, TOut], + params: Mapping[str, str], + caller: UserAPIKeyAuth, +) -> tuple[Predicate, ...] | ProblemDetail: + scope_predicates: Final = _scope_predicates(spec.scope(caller)) + if isinstance(scope_predicates, ProblemDetail): + return scope_predicates + filters: Final = _parse_filters(spec, params) + if isinstance(filters, ProblemDetail): + return filters + search: Final = _search_predicate(spec, params) + return scope_predicates + filters + ((search,) if search is not None else ()) + + +async def handle_facet( + spec: ListSpec[TRow, TOut], + executor: FacetExecutor, + request: Request, + caller: UserAPIKeyAuth, + field: str, +) -> FacetListResponse: + """The distinct values one column takes over a filtered query on a resource. + + Carries the parent's parameters so a filter dropdown offers exactly the values the + table can show, and `has_more` rather than a total, which would cost a COUNT(*) over + the whole match set on every keystroke. + """ + params: Final = request.query_params + unknown: Final = tuple(sorted(name for name in params if name == SORT_PARAM or not _is_known_param(spec, name))) + if unknown: + raise ManagementProblem(unknown_query_param_problem(unknown=unknown, allowed=_facet_allowed_params(spec))) + + duplicates: Final = _duplicate_params(request) + if duplicates: + raise ManagementProblem( + _problem( + "duplicate-query-parameter", + "Duplicate query parameter", + 400, + f"Repeated query parameter(s): {', '.join(duplicates)}. Each may appear once; " + f"use a comma-separated list for multiple filter values.", + ) + ) + + page: Final = _parse_page(params) + if isinstance(page, ProblemDetail): + raise ManagementProblem(page) + page_size: Final = _parse_page_size(spec, params) + if isinstance(page_size, ProblemDetail): + raise ManagementProblem(page_size) + + where: Final = _facet_where(spec, params, caller) + if isinstance(where, ProblemDetail): + raise ManagementProblem(where) + + values: Final = await executor.distinct(field, where) + skip: Final = (page - 1) * page_size + window: Final = values[skip : skip + page_size + 1] + has_more: Final = len(window) > page_size + return FacetListResponse( + data=tuple(window[:page_size]), + meta=PageMeta(page=page, page_size=page_size, has_more=has_more), + links=build_page_links(request=request, page=page, has_more=has_more), + ) + + def _duplicate_params(request: Request) -> tuple[str, ...]: names: Final = tuple(name for name, _ in request.query_params.multi_items()) return tuple(sorted(frozenset(name for name in names if names.count(name) > 1))) diff --git a/litellm/proxy/public_endpoints/public_v1/model_hub.py b/litellm/proxy/public_endpoints/public_v1/model_hub.py index 5a2d8068af7..b0e688740e4 100644 --- a/litellm/proxy/public_endpoints/public_v1/model_hub.py +++ b/litellm/proxy/public_endpoints/public_v1/model_hub.py @@ -3,7 +3,7 @@ from collections.abc import Mapping, Sequence from dataclasses import dataclass from types import MappingProxyType -from typing import Annotated, Final, Protocol +from typing import Annotated, Final, Literal, Protocol from fastapi import APIRouter, Depends, Request from typing_extensions import ReadOnly, TypedDict @@ -20,10 +20,12 @@ from litellm.proxy.list_api.list_framework import ( Scope, ScopeAll, SortKey, + handle_facet, handle_list, ) from litellm.proxy.utils import PrismaClient from litellm.types.proxy.management_endpoints.management_v1 import ( + FacetListResponse, ListResponse, ProblemDetail, ) @@ -95,16 +97,37 @@ class HealthEnricher: return tuple(_with_health(row, health.get(row.model_group)) for row in rows) +FEATURE_PREFIX: Final = "supports_" + + +def _features(row: ModelGroupInfoProxy) -> tuple[str, ...]: + """A row's capabilities as one repeated field, so selecting two of them matches either. + + The hub's feature control has always been a multi-select over the `supports_*` flags. + One boolean filter per flag would AND them, which is the opposite of what it does. + """ + return tuple( + sorted( + name.removeprefix(FEATURE_PREFIX) + for name, value in row.model_dump().items() + if name.startswith(FEATURE_PREFIX) and value is True + ) + ) + + def _cells(row: ModelGroupInfoProxy) -> Cells: return MappingProxyType( { "model_group": row.model_group, "mode": row.mode, "providers": tuple(row.providers), + "features": _features(row), "max_input_tokens": row.max_input_tokens, "max_output_tokens": row.max_output_tokens, "input_cost_per_token": row.input_cost_per_token, "output_cost_per_token": row.output_cost_per_token, + "rpm": row.rpm, + "tpm": row.tpm, } ) @@ -126,20 +149,28 @@ def _scope(_caller: UserAPIKeyAuth) -> Scope: MODEL_HUB_FILTERS: Final[Mapping[str, FilterSpec]] = MappingProxyType( { "mode": FilterSpec(type=str, ops=frozenset(("eq", "in"))), - "providers": FilterSpec(type=str, ops=frozenset(("contains",))), + "providers": FilterSpec(type=str, ops=frozenset(("contains", "in"))), + "features": FilterSpec(type=str, ops=frozenset(("in",))), } ) +MODEL_HUB_FACETS: Final[Mapping[str, str]] = MappingProxyType( + {"providers": "providers", "modes": "mode", "features": "features"} +) + MODEL_HUB_LIST_SPEC: Final[ListSpec[ModelGroupInfoProxy, ModelGroupInfoProxy]] = ListSpec( resource="model groups", sortable=frozenset( ( "model_group", "mode", + "providers", "max_input_tokens", "max_output_tokens", "input_cost_per_token", "output_cost_per_token", + "rpm", + "tpm", ) ), searchable=frozenset(("model_group",)), @@ -153,6 +184,32 @@ MODEL_HUB_LIST_SPEC: Final[ListSpec[ModelGroupInfoProxy, ModelGroupInfoProxy]] = ) +def _published_rows() -> Sequence[ModelGroupInfoProxy]: + from litellm.proxy.proxy_server import ( + _get_model_group_info, # pyright: ignore[reportPrivateUsage] # /public/model_hub imports it the same way + llm_router, + ) + + if llm_router is None: + raise ManagementProblem( + ProblemDetail( + type=f"{PROBLEM_TYPE_BASE}no-llm-router", + title="No models configured", + status=400, + detail=CommonProxyErrors.no_llm_router.value, + ) + ) + if litellm.public_model_groups is None: + return () + return tuple( + _get_model_group_info( + llm_router=llm_router, + all_models_str=litellm.public_model_groups, + model_group=None, + ) + ) + + def _executor( rows: Sequence[ModelGroupInfoProxy], prisma_client: PrismaClient | None, @@ -191,37 +248,11 @@ async def public_model_hub_list( ``` """ try: - from litellm.proxy.proxy_server import ( - _get_model_group_info, # pyright: ignore[reportPrivateUsage] # /public/model_hub imports it the same way - llm_router, - prisma_client, - ) - - if llm_router is None: - raise ManagementProblem( - ProblemDetail( - type=f"{PROBLEM_TYPE_BASE}no-llm-router", - title="No models configured", - status=400, - detail=CommonProxyErrors.no_llm_router.value, - ) - ) - - rows: Final[Sequence[ModelGroupInfoProxy]] = ( - () - if litellm.public_model_groups is None - else tuple( - _get_model_group_info( - llm_router=llm_router, - all_models_str=litellm.public_model_groups, - model_group=None, - ) - ) - ) + from litellm.proxy.proxy_server import prisma_client return await handle_list( spec=MODEL_HUB_LIST_SPEC, - executor=_executor(rows, prisma_client), + executor=_executor(_published_rows(), prisma_client), request=request, caller=user_api_key_dict, ) @@ -240,3 +271,53 @@ async def public_model_hub_list( detail="Failed to list public model groups.", ) ) + + +@router.get( + "/model_hub/{facet}", + tags=["public", "model management"], # mutable-ok: fastapi types tags as list[str | Enum] + dependencies=(Depends(user_api_key_auth),), + response_model=FacetListResponse, +) +async def public_model_hub_facet( + request: Request, + facet: Literal["providers", "modes", "features"], + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +) -> FacetListResponse: + """ + The distinct providers, modes or features across the published model groups, for the + Model Hub's filter dropdowns. No authentication. + + Carries the same filters and search as the list route, so a dropdown offers exactly + the values the table can show: asking for providers under `filter[mode][in]=chat` + lists only the providers that serve a chat model. + + Example curl: + ``` + curl --location --globoff \ + 'http://0.0.0.0:4000/public/v1/model_hub/providers?filter[mode][in]=chat&page_size=50' + ``` + """ + try: + return await handle_facet( + spec=MODEL_HUB_LIST_SPEC, + executor=InMemoryListExecutor(rows=_published_rows(), cells=_cells), + request=request, + caller=user_api_key_dict, + field=MODEL_HUB_FACETS[facet], + ) + + except ManagementProblem: + raise + except Exception as e: # noqa: BLE001 # a router error answers as a problem document, not the OpenAI error shape + verbose_proxy_logger.exception( + "litellm.proxy.public_endpoints.public_v1.model_hub.public_model_hub_facet(): Exception occured - %s", e + ) + raise ManagementProblem( + ProblemDetail( + type=f"{PROBLEM_TYPE_BASE}internal-server-error", + title="Internal server error", + status=500, + detail="Failed to list public model group values.", + ) + ) diff --git a/litellm/types/proxy/management_endpoints/management_v1.py b/litellm/types/proxy/management_endpoints/management_v1.py index b2244f6eb9b..aa82138110d 100644 --- a/litellm/types/proxy/management_endpoints/management_v1.py +++ b/litellm/types/proxy/management_endpoints/management_v1.py @@ -1,5 +1,6 @@ """Shared response shapes for the `/management/v1` control-plane surface.""" +from collections.abc import Sequence from typing import Generic, TypeVar from pydantic import BaseModel, ConfigDict, Field @@ -38,7 +39,7 @@ class PageMeta(BaseModel): class FacetListResponse(BaseModel): """The distinct values one column takes over a filtered query. `data` holds bare values, not entity rows.""" - data: list[str] + data: Sequence[str] meta: PageMeta links: PageLinks diff --git a/tests/test_litellm/proxy/list_api/test_list_framework.py b/tests/test_litellm/proxy/list_api/test_list_framework.py index 6ed3ab369c2..ab7ef229e1a 100644 --- a/tests/test_litellm/proxy/list_api/test_list_framework.py +++ b/tests/test_litellm/proxy/list_api/test_list_framework.py @@ -25,6 +25,7 @@ from litellm.proxy.list_api.list_framework import ( SortKey, Within, build_query_plan, + handle_facet, handle_list, order_by_sql, where_sql, @@ -892,3 +893,107 @@ def test_the_facet_page_shapes_are_untouched_by_page_mode(): assert set(links) == {"self", "prev", "next"} assert links["next"] == "/management/v1/budgets?q=ac&page=3" + + +# ------------------------------------------------------- facet request handling + + +class RecordingFacetExecutor: + """Records the one call `handle_facet` is allowed to make, so a rejected request + can be shown never to have reached it.""" + + def __init__(self, values: tuple[str, ...] = ()) -> None: + self.values = values + self.field: str | None = None + self.where: tuple[object, ...] | None = None + + async def distinct(self, field: str, where: tuple[object, ...]) -> Sequence[str]: + self.field = field + self.where = where + return self.values + + +async def _facet_problem(query: str, spec: ListSpec[BudgetRow, BudgetOut] | None = None) -> ProblemDetail: + executor = RecordingFacetExecutor(values=("a", "b")) + with pytest.raises(ManagementProblem) as raised: + await handle_facet( + spec=spec or _spec(), + executor=executor, + request=_request(query), + caller=CALLER, + field="created_by", + ) + assert executor.field is None, "a rejected facet request still queried the executor" + return raised.value.problem + + +@pytest.mark.asyncio +async def test_a_facet_conjoins_the_scope_with_the_callers_filters(): + """The scope is the one predicate a caller cannot drop, so a facet has to add to it + rather than replace it: otherwise a dropdown lists values from rows the caller + cannot see in the table.""" + spec = _spec(scope=lambda caller: ScopeWhere(where=(Compare(field="created_by", op="eq", value="caller-1"),))) + executor = RecordingFacetExecutor(values=("caller-1",)) + + response = await handle_facet( + spec=spec, + executor=executor, + request=_request("filter[max_budget][gte]=5&q=ac"), + caller=CALLER, + field="created_by", + ) + + assert tuple(response.data) == ("caller-1",) + assert executor.where == ( + Compare(field="created_by", op="eq", value="caller-1"), + Compare(field="max_budget", op="gte", value=5.0), + AnyOf( + clauses=( + Compare(field="budget_id", op="contains", value="ac"), + Compare(field="created_by", op="contains", value="ac"), + ) + ), + ) + + +@pytest.mark.asyncio +async def test_a_denied_scope_on_a_facet_never_reaches_the_executor(): + """A 200 with an empty list would read as "no such values" rather than "not yours".""" + problem = await _facet_problem("", spec=_spec(scope=lambda caller: ScopeDenied(reason="nope"))) + + assert problem.status == 403 + assert problem.type == f"{PROBLEM_TYPE_BASE}forbidden" + + +@pytest.mark.asyncio +async def test_a_facet_rejects_a_filter_operator_its_spec_does_not_offer(): + problem = await _facet_problem("filter[created_by][gte]=x") + + assert problem.status == 400 + assert "gte" in problem.detail + + +@pytest.mark.asyncio +async def test_a_facet_rejects_a_repeated_query_parameter(): + problem = await _facet_problem("page=1&page=2") + + assert problem.type == f"{PROBLEM_TYPE_BASE}duplicate-query-parameter" + assert "page" in problem.detail + + +@pytest.mark.asyncio +@pytest.mark.parametrize("query", ("page=0", "page=one")) +async def test_a_facet_rejects_a_page_that_is_not_a_positive_integer(query: str): + problem = await _facet_problem(query) + + assert problem.status == 400 + assert problem.type == f"{PROBLEM_TYPE_BASE}invalid-query-parameter" + assert "'page'" in problem.detail + + +@pytest.mark.asyncio +async def test_a_facet_rejects_a_page_size_that_is_not_a_positive_integer(): + problem = await _facet_problem("page_size=0") + + assert problem.status == 400 + assert "'page_size'" in problem.detail diff --git a/tests/test_litellm/proxy/public_endpoints/public_v1/test_model_hub.py b/tests/test_litellm/proxy/public_endpoints/public_v1/test_model_hub.py index 631e91dca11..de2d95e9f28 100644 --- a/tests/test_litellm/proxy/public_endpoints/public_v1/test_model_hub.py +++ b/tests/test_litellm/proxy/public_endpoints/public_v1/test_model_hub.py @@ -193,12 +193,12 @@ def test_sorting_by_a_numeric_field_puts_the_unset_ones_last_in_both_directions( def test_an_undeclared_sort_field_is_a_problem_naming_the_allowed_fields(monkeypatch): _publish(monkeypatch, _named(3)) - response = _get("sort=providers") + response = _get("sort=health_status") assert response.status_code == 400 assert response.headers["content-type"].startswith("application/problem+json") body = response.json() - assert "providers" in body["detail"] + assert "health_status" in body["detail"] assert body["allowed"] == [ "input_cost_per_token", "max_input_tokens", @@ -206,6 +206,9 @@ def test_an_undeclared_sort_field_is_a_problem_naming_the_allowed_fields(monkeyp "mode", "model_group", "output_cost_per_token", + "providers", + "rpm", + "tpm", ] @@ -347,3 +350,131 @@ def test_the_endpoint_it_supersedes_still_answers_with_its_bare_array(monkeypatc body = response.json() assert isinstance(body, list) assert [row["model_group"] for row in body] == ["model-000", "model-001", "model-002"] + + +FACET_PATHS = ("providers", "modes", "features") + + +def _facet(name: str, query: str = ""): + suffix = f"?{query}" if query else "" + return client.get(f"{MODEL_HUB_PATH}/{name}{suffix}") + + +def test_providers_filter_accepts_several_providers_at_once(monkeypatch): + """The hub's provider control is a multi-select, so the route has to OR the values.""" + _publish( + monkeypatch, + ( + _info("gpt-4", providers=("openai",)), + _info("claude", providers=("anthropic",)), + _info("mistral-large", providers=("mistral",)), + _info("router", providers=("openai", "anthropic")), + ), + ) + + response = _get("filter[providers][in]=openai,anthropic") + + assert response.status_code == 200, response.text + assert sorted(_groups(response)) == ["claude", "gpt-4", "router"] + + +def test_features_filter_matches_a_model_with_any_of_the_named_features(monkeypatch): + """Selecting two features widens the result set, the way the hub's multi-select always did.""" + _publish( + monkeypatch, + ( + _info("sees", supports_vision=True), + _info("calls", supports_function_calling=True), + _info("both", supports_vision=True, supports_function_calling=True), + _info("plain"), + ), + ) + + response = _get("filter[features][in]=vision,function_calling") + + assert response.status_code == 200, response.text + assert sorted(_groups(response)) == ["both", "calls", "sees"] + + +def test_a_single_feature_filter_selects_only_models_with_it(monkeypatch): + _publish(monkeypatch, (_info("sees", supports_vision=True), _info("plain"), _info("reasons", supports_reasoning=True))) + + assert _groups(_get("filter[features][in]=vision")) == ["sees"] + assert _groups(_get("filter[features][in]=reasoning")) == ["reasons"] + + +def test_providers_and_limits_are_sortable(monkeypatch): + """The hub sorted on these columns before it paged; they stay sortable now that the route orders.""" + _publish( + monkeypatch, + ( + _info("b-model", providers=("mistral",), rpm=10), + _info("a-model", providers=("anthropic",), rpm=30), + _info("c-model", providers=("openai",), rpm=20), + ), + ) + + assert _groups(_get("sort=providers")) == ["a-model", "b-model", "c-model"] + assert _groups(_get("sort=-rpm")) == ["a-model", "c-model", "b-model"] + + +@pytest.mark.parametrize("facet", FACET_PATHS) +def test_a_facet_serves_the_distinct_values_of_its_column(monkeypatch, facet): + _publish( + monkeypatch, + ( + _info("a", providers=("openai",), mode="chat", supports_vision=True), + _info("b", providers=("anthropic", "openai"), mode="embedding", supports_vision=True), + _info("c", providers=("mistral",), mode="chat"), + ), + ) + + response = _facet(facet) + + assert response.status_code == 200, response.text + assert response.json()["data"] == { + "providers": ["anthropic", "mistral", "openai"], + "modes": ["chat", "embedding"], + "features": ["vision"], + }[facet] + + +def test_a_facet_offers_only_values_the_table_can_show(monkeypatch): + """Section 12's reason for hanging facets off the resource: the dropdown matches the filtered table.""" + _publish( + monkeypatch, + ( + _info("chat-openai", providers=("openai",), mode="chat"), + _info("embed-cohere", providers=("cohere",), mode="embedding"), + ), + ) + + assert _facet("providers", "filter[mode][in]=chat").json()["data"] == ["openai"] + assert _facet("providers", "q=embed").json()["data"] == ["cohere"] + + +def test_a_facet_pages_and_reports_whether_more_remain(monkeypatch): + _publish(monkeypatch, tuple(_info(f"m-{index}", providers=(f"p-{index:02d}",)) for index in range(5))) + + first = _facet("providers", "page_size=2") + last = _facet("providers", "page=3&page_size=2") + + assert first.json()["data"] == ["p-00", "p-01"] + assert first.json()["meta"] == {"page": 1, "page_size": 2, "has_more": True} + assert last.json()["data"] == ["p-04"] + assert last.json()["meta"]["has_more"] is False + + +def test_a_facet_rejects_a_sort_it_does_not_offer(monkeypatch): + """Facet values are always ascending, so `sort` is not part of the facet contract.""" + _publish(monkeypatch, _named(3)) + + response = _facet("providers", "sort=-providers") + + assert response.status_code == 400 + assert response.json()["type"].endswith("unknown-query-parameter") + + +@pytest.mark.parametrize("facet", FACET_PATHS) +def test_a_facet_is_reachable_without_a_key(facet): + assert f"{MODEL_HUB_PATH}/{facet}" in LiteLLMRoutes.public_routes.value diff --git a/ui/litellm-dashboard/src/components/PublicModelHubTableColumns.tsx b/ui/litellm-dashboard/src/components/PublicModelHubTableColumns.tsx index ab0ed976149..de987360791 100644 --- a/ui/litellm-dashboard/src/components/PublicModelHubTableColumns.tsx +++ b/ui/litellm-dashboard/src/components/PublicModelHubTableColumns.tsx @@ -6,6 +6,7 @@ import { DataTableSortHeader } from "@/components/shared/DataTable"; import { CellTooltip, IdentityCell, StatusBadge, type StatusTone } from "@/components/shared/table_cells"; import { Badge } from "@/components/ui/badge"; import { getProviderLogoAndName } from "@/components/provider_info_helpers"; +import { PUBLIC_MODEL_HUB_SORTABLE_FIELDS } from "@/components/publicModelHub/publicModelHubFilters"; export interface ModelGroupInfo { model_group: string; @@ -163,154 +164,150 @@ interface PublicModelHubColumnsDeps { onModelClick: (model: ModelGroupInfo) => void; } -export const getPublicModelHubColumns = ({ onModelClick }: PublicModelHubColumnsDeps): ColumnDef[] => [ - { - id: "model_group", - accessorKey: "model_group", - meta: { title: "Model Name" }, - header: ({ column }) => , - size: 200, - enableSorting: true, - sortingFn: "alphanumeric", - cell: ({ row }) => ( - onModelClick(row.original)} - /> - ), - }, - { - id: "providers", - accessorKey: "providers", - meta: { title: "Providers", skeleton: "chips" }, - header: ({ column }) => , - size: 150, - enableSorting: true, - sortingFn: (rowA, rowB) => - (rowA.original.providers ?? []).join(", ").localeCompare((rowB.original.providers ?? []).join(", ")), - cell: ({ row }) => , - }, - { - id: "mode", - accessorKey: "mode", - meta: { title: "Mode" }, - header: ({ column }) => , - size: 110, - enableSorting: true, - sortingFn: "alphanumeric", - cell: ({ row }) => ( - - {getModeIcon(row.original.mode || "")} - {row.original.mode || "Chat"} - - ), - }, - { - id: "max_input_tokens", - accessorKey: "max_input_tokens", - meta: { title: "Max Input", numeric: true }, - header: ({ column }) => , - size: 100, - enableSorting: true, - cell: ({ row }) => {formatTokens(row.original.max_input_tokens)}, - }, - { - id: "max_output_tokens", - accessorKey: "max_output_tokens", - meta: { title: "Max Output", numeric: true }, - header: ({ column }) => , - size: 100, - enableSorting: true, - cell: ({ row }) => {formatTokens(row.original.max_output_tokens)}, - }, - { - id: "input_cost_per_token", - accessorKey: "input_cost_per_token", - meta: { title: "Input $/1M", numeric: true }, - header: ({ column }) => , - size: 110, - enableSorting: true, - cell: ({ row }) => ( - - {row.original.input_cost_per_token ? formatCost(row.original.input_cost_per_token) : "Free"} - - ), - }, - { - id: "output_cost_per_token", - accessorKey: "output_cost_per_token", - meta: { title: "Output $/1M", numeric: true }, - header: ({ column }) => , - size: 110, - enableSorting: true, - cell: ({ row }) => ( - - {row.original.output_cost_per_token ? formatCost(row.original.output_cost_per_token) : "Free"} - - ), - }, - { - id: "features", - meta: { title: "Features", skeleton: "chips" }, - header: "Features", - size: 140, - enableSorting: false, - cell: ({ row }) => { - const features = Object.entries(row.original) - .filter(([key, value]) => key.startsWith("supports_") && value === true) - .map(([key]) => formatCapabilityName(key)); - return ; - }, - }, - { - id: "health_status", - accessorKey: "health_status", - meta: { title: "Health Status", skeleton: "badge" }, - header: ({ column }) => , - size: 130, - enableSorting: true, - cell: ({ row }) => { - const model = row.original; - const responseTimeLabel = model.health_response_time - ? `Response Time: ${Number(model.health_response_time).toFixed(2)}ms` - : "N/A"; - const lastCheckedLabel = model.health_checked_at - ? `Last Checked: ${new Date(model.health_checked_at).toLocaleString()}` - : "N/A"; - return ( - -
{responseTimeLabel}
-
{lastCheckedLabel}
- - } - trigger={ - - - - } +export const getPublicModelHubColumns = ({ onModelClick }: PublicModelHubColumnsDeps): ColumnDef[] => { + const columns: ColumnDef[] = [ + { + id: "model_group", + accessorKey: "model_group", + meta: { title: "Model Name" }, + header: ({ column }) => , + size: 200, + sortingFn: "alphanumeric", + cell: ({ row }) => ( + onModelClick(row.original)} /> - ); + ), }, - }, - { - id: "rpm", - accessorKey: "rpm", - meta: { title: "Limits" }, - header: ({ column }) => , - size: 150, - enableSorting: true, - cell: ({ row }) => ( - {formatLimits(row.original.rpm, row.original.tpm)} - ), - }, -]; + { + id: "providers", + accessorKey: "providers", + meta: { title: "Providers", skeleton: "chips" }, + header: ({ column }) => , + size: 150, + sortingFn: (rowA, rowB) => + (rowA.original.providers ?? []).join(", ").localeCompare((rowB.original.providers ?? []).join(", ")), + cell: ({ row }) => , + }, + { + id: "mode", + accessorKey: "mode", + meta: { title: "Mode" }, + header: ({ column }) => , + size: 110, + sortingFn: "alphanumeric", + cell: ({ row }) => ( + + {getModeIcon(row.original.mode || "")} + {row.original.mode || "Chat"} + + ), + }, + { + id: "max_input_tokens", + accessorKey: "max_input_tokens", + meta: { title: "Max Input", numeric: true }, + header: ({ column }) => , + size: 100, + cell: ({ row }) => {formatTokens(row.original.max_input_tokens)}, + }, + { + id: "max_output_tokens", + accessorKey: "max_output_tokens", + meta: { title: "Max Output", numeric: true }, + header: ({ column }) => , + size: 100, + cell: ({ row }) => {formatTokens(row.original.max_output_tokens)}, + }, + { + id: "input_cost_per_token", + accessorKey: "input_cost_per_token", + meta: { title: "Input $/1M", numeric: true }, + header: ({ column }) => , + size: 110, + cell: ({ row }) => ( + + {row.original.input_cost_per_token ? formatCost(row.original.input_cost_per_token) : "Free"} + + ), + }, + { + id: "output_cost_per_token", + accessorKey: "output_cost_per_token", + meta: { title: "Output $/1M", numeric: true }, + header: ({ column }) => , + size: 110, + cell: ({ row }) => ( + + {row.original.output_cost_per_token ? formatCost(row.original.output_cost_per_token) : "Free"} + + ), + }, + { + id: "features", + meta: { title: "Features", skeleton: "chips" }, + header: "Features", + size: 140, + cell: ({ row }) => { + const features = Object.entries(row.original) + .filter(([key, value]) => key.startsWith("supports_") && value === true) + .map(([key]) => formatCapabilityName(key)); + return ; + }, + }, + { + id: "health_status", + accessorKey: "health_status", + meta: { title: "Health Status", skeleton: "badge" }, + header: ({ column }) => , + size: 130, + cell: ({ row }) => { + const model = row.original; + const responseTimeLabel = model.health_response_time + ? `Response Time: ${Number(model.health_response_time).toFixed(2)}ms` + : "N/A"; + const lastCheckedLabel = model.health_checked_at + ? `Last Checked: ${new Date(model.health_checked_at).toLocaleString()}` + : "N/A"; + return ( + +
{responseTimeLabel}
+
{lastCheckedLabel}
+ + } + trigger={ + + + + } + /> + ); + }, + }, + { + id: "rpm", + accessorKey: "rpm", + meta: { title: "Limits" }, + header: ({ column }) => , + size: 150, + cell: ({ row }) => ( + {formatLimits(row.original.rpm, row.original.tpm)} + ), + }, + ]; + return columns.map((column) => ({ + ...column, + enableSorting: PUBLIC_MODEL_HUB_SORTABLE_FIELDS.includes(String(column.id)), + })); +}; interface PublicAgentHubColumnsDeps { onAgentClick: (agent: AgentCard) => void; diff --git a/ui/litellm-dashboard/src/components/publicModelHub/publicModelHubFilters.test.ts b/ui/litellm-dashboard/src/components/publicModelHub/publicModelHubFilters.test.ts new file mode 100644 index 00000000000..d17f1ccae10 --- /dev/null +++ b/ui/litellm-dashboard/src/components/publicModelHub/publicModelHubFilters.test.ts @@ -0,0 +1,85 @@ +import type { ColumnFiltersState } from "@tanstack/react-table"; +import { describe, expect, it } from "vitest"; + +import { + FEATURE_FILTER_ID, + MODE_FILTER_ID, + PROVIDER_FILTER_ID, + featureLabel, + readFilterValues, + serializePublicModelHubFilters, + withFilterValue, +} from "./publicModelHubFilters"; + +describe("serializePublicModelHubFilters", () => { + it("sends each multi-select as the route's comma separated in filter", () => { + const filters: ColumnFiltersState = [ + { id: MODE_FILTER_ID, value: ["chat", "embedding"] }, + { id: PROVIDER_FILTER_ID, value: ["openai", "anthropic"] }, + { id: FEATURE_FILTER_ID, value: ["vision"] }, + ]; + + expect(serializePublicModelHubFilters(filters)).toEqual({ + "filter[mode][in]": "chat,embedding", + "filter[providers][in]": "openai,anthropic", + "filter[features][in]": "vision", + }); + }); + + it("omits blank filters rather than sending parameters the route rejects", () => { + const filters: ColumnFiltersState = [ + { id: MODE_FILTER_ID, value: [] }, + { id: PROVIDER_FILTER_ID, value: [] }, + ]; + + expect(serializePublicModelHubFilters(filters)).toEqual({}); + }); + + it("ignores filter ids the route does not declare", () => { + expect(serializePublicModelHubFilters([{ id: "health_status", value: ["healthy"] }])).toEqual({}); + }); +}); + +describe("readFilterValues", () => { + it("reads back the values of the filter it names", () => { + const filters: ColumnFiltersState = [ + { id: MODE_FILTER_ID, value: ["chat"] }, + { id: FEATURE_FILTER_ID, value: ["vision", "reasoning"] }, + ]; + + expect(readFilterValues(filters, FEATURE_FILTER_ID)).toEqual(["vision", "reasoning"]); + expect(readFilterValues(filters, PROVIDER_FILTER_ID)).toEqual([]); + }); +}); + +describe("withFilterValue", () => { + it("adds a filter that is not set yet", () => { + expect(withFilterValue([], PROVIDER_FILTER_ID, ["openai"])).toEqual([ + { id: PROVIDER_FILTER_ID, value: ["openai"] }, + ]); + }); + + it("replaces a filter instead of stacking a second one", () => { + const filters: ColumnFiltersState = [{ id: PROVIDER_FILTER_ID, value: ["openai"] }]; + + expect(withFilterValue(filters, PROVIDER_FILTER_ID, ["anthropic"])).toEqual([ + { id: PROVIDER_FILTER_ID, value: ["anthropic"] }, + ]); + }); + + it("drops a cleared filter and leaves the others alone", () => { + const filters: ColumnFiltersState = [ + { id: MODE_FILTER_ID, value: ["chat"] }, + { id: PROVIDER_FILTER_ID, value: ["openai"] }, + ]; + + expect(withFilterValue(filters, PROVIDER_FILTER_ID, [])).toEqual([{ id: MODE_FILTER_ID, value: ["chat"] }]); + }); +}); + +describe("featureLabel", () => { + it("renders a route feature the way the hub has always labelled it", () => { + expect(featureLabel("vision")).toBe("Vision"); + expect(featureLabel("parallel_function_calling")).toBe("Parallel Function Calling"); + }); +}); diff --git a/ui/litellm-dashboard/src/components/publicModelHub/publicModelHubFilters.ts b/ui/litellm-dashboard/src/components/publicModelHub/publicModelHubFilters.ts new file mode 100644 index 00000000000..b217fdae955 --- /dev/null +++ b/ui/litellm-dashboard/src/components/publicModelHub/publicModelHubFilters.ts @@ -0,0 +1,60 @@ +import type { ColumnFilter, ColumnFiltersState } from "@tanstack/react-table"; + +export const MODE_FILTER_ID = "mode"; +export const PROVIDER_FILTER_ID = "providers"; +export const FEATURE_FILTER_ID = "features"; + +export const PUBLIC_MODEL_HUB_SORTABLE_FIELDS: readonly string[] = [ + "model_group", + "mode", + "providers", + "max_input_tokens", + "max_output_tokens", + "input_cost_per_token", + "output_cost_per_token", + "rpm", + "tpm", +]; + +type QueryEntry = readonly [string, string]; + +type FilterValue = string | string[]; + +const entries = (key: string, value: string): QueryEntry[] => (value === "" ? [] : [[key, value]]); + +const asStringArray = (value: unknown): string[] => + Array.isArray(value) ? value.filter((item): item is string => typeof item === "string") : []; + +const inFilter = (field: string, value: unknown): QueryEntry[] => + entries(`filter[${field}][in]`, asStringArray(value).join(",")); + +const filterParams = (filter: ColumnFilter): QueryEntry[] => { + switch (filter.id) { + case MODE_FILTER_ID: + case PROVIDER_FILTER_ID: + case FEATURE_FILTER_ID: + return inFilter(filter.id, filter.value); + default: + return []; + } +}; + +export const serializePublicModelHubFilters = (filters: ColumnFiltersState): Readonly> => + Object.fromEntries(filters.flatMap(filterParams)); + +export const readFilterValues = (filters: ColumnFiltersState, id: string): string[] => + asStringArray(filters.find((filter) => filter.id === id)?.value); + +const isEmpty = (value: FilterValue): boolean => (Array.isArray(value) ? value.length === 0 : value.trim() === ""); + +export const withFilterValue = (filters: ColumnFiltersState, id: string, value: FilterValue): ColumnFiltersState => { + const others = filters.filter((filter) => filter.id !== id); + return isEmpty(value) ? others : [...others, { id, value }]; +}; + +/** `supports_vision` reaches the route as `vision`; the hub has always shown it as "Vision". */ +export const featureLabel = (feature: string): string => + feature + .split("_") + .map((word) => word.charAt(0).toUpperCase() + word.slice(1)) + .join(" "); diff --git a/ui/litellm-dashboard/src/components/publicModelHub/usePublicModelHubFacets.ts b/ui/litellm-dashboard/src/components/publicModelHub/usePublicModelHubFacets.ts new file mode 100644 index 00000000000..d6b356ac8f9 --- /dev/null +++ b/ui/litellm-dashboard/src/components/publicModelHub/usePublicModelHubFacets.ts @@ -0,0 +1,47 @@ +"use client"; + +import { useQueries } from "@tanstack/react-query"; + +import { apiClient } from "@/components/networking"; +import type { components } from "@/lib/http/schema"; + +import { PUBLIC_MODEL_HUB_PATH } from "./usePublicModelHubList"; + +type FacetResponse = components["schemas"]["FacetListResponse"]; + +export const MODEL_HUB_FACETS = ["providers", "modes", "features"] as const; + +export type ModelHubFacet = (typeof MODEL_HUB_FACETS)[number]; + +/** The route caps a page at 100, which is far above the distinct providers, modes or features any proxy publishes. */ +const FACET_PAGE_SIZE = 100; + +export interface PublicModelHubFacets { + providers: string[]; + modes: string[]; + features: string[]; +} + +const fetchFacet = (facet: ModelHubFacet, signal: AbortSignal): Promise => + apiClient.get(`${PUBLIC_MODEL_HUB_PATH}/${facet}`, { + query: { page_size: FACET_PAGE_SIZE }, + signal, + }); + +/** + * The values each filter dropdown offers, read from the route rather than derived from a + * page of rows, which can only ever show the values that page happens to contain. + */ +export const usePublicModelHubFacets = (enabled: boolean): PublicModelHubFacets => { + const results = useQueries({ + queries: MODEL_HUB_FACETS.map((facet) => ({ + queryKey: ["publicModelHub", "facet", facet], + queryFn: ({ signal }: { signal: AbortSignal }) => fetchFacet(facet, signal), + enabled, + staleTime: Infinity, + })), + }); + + const [providers, modes, features] = results.map((result) => result.data?.data ?? []); + return { providers, modes, features }; +}; diff --git a/ui/litellm-dashboard/src/components/publicModelHub/usePublicModelHubList.ts b/ui/litellm-dashboard/src/components/publicModelHub/usePublicModelHubList.ts new file mode 100644 index 00000000000..2be66a010d1 --- /dev/null +++ b/ui/litellm-dashboard/src/components/publicModelHub/usePublicModelHubList.ts @@ -0,0 +1,83 @@ +"use client"; + +import type { SortingState } from "@tanstack/react-table"; +import { useCallback } from "react"; + +import { + useResourceList, + type ResourceListPage, + type ResourceListQuery, + type ResourceListResult, +} from "@/app/(dashboard)/hooks/common/useResourceList"; +import { apiClient } from "@/components/networking"; +import type { ModelGroupInfo } from "@/components/PublicModelHubTableColumns"; + +import { + FEATURE_FILTER_ID, + MODE_FILTER_ID, + PROVIDER_FILTER_ID, + readFilterValues, + serializePublicModelHubFilters, + withFilterValue, +} from "./publicModelHubFilters"; + +export const PUBLIC_MODEL_HUB_PATH = "/public/v1/model_hub"; +export const PUBLIC_MODEL_HUB_PAGE_SIZE = 50; + +const QUERY_KEY = ["publicModelHub", "list"] as const; +const DEFAULT_SORTING: SortingState = [{ id: "model_group", desc: false }]; + +export interface PublicModelHubListResult extends ResourceListResult { + providerValues: string[]; + onProvidersChange: (values: string[]) => void; + modeValues: string[]; + onModesChange: (values: string[]) => void; + featureValues: string[]; + onFeaturesChange: (values: string[]) => void; + hasActiveQuery: boolean; +} + +const fetchPage = async (query: ResourceListQuery, signal: AbortSignal): Promise> => { + try { + return await apiClient.get>(PUBLIC_MODEL_HUB_PATH, { query, signal }); + } catch (error) { + if (!signal.aborted) { + console.error("There was an error fetching the public model data", error); + } + throw error; + } +}; + +export const usePublicModelHubList = (enabled: boolean): PublicModelHubListResult => { + const listOptions = { + queryKey: QUERY_KEY, + fetchPage, + serializeFilters: serializePublicModelHubFilters, + defaultSorting: DEFAULT_SORTING, + defaultPageSize: PUBLIC_MODEL_HUB_PAGE_SIZE, + enabled, + }; + const list = useResourceList(listOptions); + + const { onColumnFiltersChange } = list; + + const setFilter = useCallback( + (id: string, values: string[]) => onColumnFiltersChange((previous) => withFilterValue(previous, id, values)), + [onColumnFiltersChange], + ); + + const onProvidersChange = useCallback((values: string[]) => setFilter(PROVIDER_FILTER_ID, values), [setFilter]); + const onModesChange = useCallback((values: string[]) => setFilter(MODE_FILTER_ID, values), [setFilter]); + const onFeaturesChange = useCallback((values: string[]) => setFilter(FEATURE_FILTER_ID, values), [setFilter]); + + return { + ...list, + providerValues: readFilterValues(list.columnFilters, PROVIDER_FILTER_ID), + onProvidersChange, + modeValues: readFilterValues(list.columnFilters, MODE_FILTER_ID), + onModesChange, + featureValues: readFilterValues(list.columnFilters, FEATURE_FILTER_ID), + onFeaturesChange, + hasActiveQuery: list.searchValue.trim() !== "" || list.columnFilters.length > 0, + }; +}; diff --git a/ui/litellm-dashboard/src/components/public_model_hub.test.tsx b/ui/litellm-dashboard/src/components/public_model_hub.test.tsx index fec46e98077..cb23dfd9bb6 100644 --- a/ui/litellm-dashboard/src/components/public_model_hub.test.tsx +++ b/ui/litellm-dashboard/src/components/public_model_hub.test.tsx @@ -1,8 +1,12 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; import { describe, it, expect, vi, beforeAll, beforeEach } from "vitest"; import { render, screen, waitFor, within, fireEvent } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; import { flexRender, getCoreRowModel, useReactTable } from "@tanstack/react-table"; import PublicModelHub from "./public_model_hub"; -import { getPublicMCPHubColumns, MCPServerData } from "./PublicModelHubTableColumns"; +import { getPublicMCPHubColumns, MCPServerData, ModelGroupInfo } from "./PublicModelHubTableColumns"; + +const { apiGetMock } = vi.hoisted(() => ({ apiGetMock: vi.fn() })); vi.mock("next/navigation", () => ({ useRouter: vi.fn(() => ({ @@ -16,6 +20,7 @@ vi.mock("./networking", async (importOriginal) => { const actual = await importOriginal(); return { ...actual, + apiClient: { ...actual.apiClient, get: apiGetMock }, modelHubPublicModelsCall: vi.fn().mockResolvedValue([]), getPublicModelHubInfo: vi.fn().mockResolvedValue({ docs_title: "LiteLLM Gateway", @@ -34,6 +39,68 @@ vi.mock("./navbar", () => ({ default: vi.fn(() =>
Navbar Component
), })); +const MODEL_HUB_PATH = "/public/v1/model_hub"; + +const FACET_VALUES: Record = { + [`${MODEL_HUB_PATH}/providers`]: ["anthropic", "openai"], + [`${MODEL_HUB_PATH}/modes`]: ["chat", "embedding"], + [`${MODEL_HUB_PATH}/features`]: ["function_calling", "vision"], +}; + +const MODEL_DEFAULTS = { + providers: ["openai"], + mode: "chat", + supports_function_calling: false, + supports_vision: false, + supports_parallel_function_calling: false, +}; + +const model = (overrides: Partial & { model_group: string }): ModelGroupInfo => ({ + ...MODEL_DEFAULTS, + ...overrides, +}); + +const DEFAULT_MODELS = [model({ model_group: "gpt-4" }), model({ model_group: "claude-3", providers: ["anthropic"] })]; + +const respondWith = (rows: ModelGroupInfo[], totalCount: number = rows.length, pageSize: number = 50) => + apiGetMock.mockImplementation((path: string) => { + const facet = FACET_VALUES[path]; + if (facet) { + return Promise.resolve({ + data: facet, + meta: { page: 1, page_size: 100, has_more: false }, + links: { self: path, prev: null, next: null }, + }); + } + return Promise.resolve({ + data: rows, + meta: { + total_count: totalCount, + page: 1, + page_size: pageSize, + total_pages: Math.max(Math.ceil(totalCount / pageSize), 1), + }, + links: { self: MODEL_HUB_PATH, first: MODEL_HUB_PATH, prev: null, next: null, last: MODEL_HUB_PATH }, + }); + }); + +type QueryRecord = Record; + +const modelCalls = () => apiGetMock.mock.calls.filter((call) => call[0] === MODEL_HUB_PATH); +const facetPaths = (): string[] => + apiGetMock.mock.calls.map((call) => String(call[0])).filter((path) => path.startsWith(`${MODEL_HUB_PATH}/`)); +const modelQueries = (): QueryRecord[] => modelCalls().map((call) => (call[1] as { query: QueryRecord }).query); +const lastModelQuery = (): QueryRecord => modelQueries()[modelQueries().length - 1]; + +const renderHub = () => { + const client = new QueryClient({ defaultOptions: { queries: { retry: false, gcTime: 0 } } }); + return render( + + + , + ); +}; + beforeAll(() => { Object.defineProperty(window, "matchMedia", { writable: true, @@ -51,6 +118,8 @@ beforeAll(() => { }); beforeEach(() => { + vi.clearAllMocks(); + respondWith(DEFAULT_MODELS); Storage.prototype.getItem = vi.fn(() => "false"); Storage.prototype.setItem = vi.fn(); Object.defineProperty(window, "location", { @@ -64,58 +133,215 @@ beforeEach(() => { describe("PublicModelHub", () => { it("renders", () => { - const { container } = render(); + const { container } = renderHub(); expect(container).toBeInTheDocument(); }); + it("loads the first page of models from the paginated public endpoint", async () => { + renderHub(); + + expect(await screen.findByText("gpt-4")).toBeInTheDocument(); + expect(modelCalls()[0][0]).toBe(MODEL_HUB_PATH); + expect(modelQueries()[0]).toEqual({ page: 1, page_size: 50, sort: "model_group" }); + }); + + it("waits for the resolved proxy base url before asking for a page", async () => { + const networkingModule = await import("./networking"); + let publishConfig: () => void = () => {}; + vi.mocked(networkingModule.getUiConfig).mockReturnValueOnce( + new Promise((resolve) => { + publishConfig = () => resolve({} as Awaited>); + }), + ); + + renderHub(); + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(modelCalls()).toHaveLength(0); + + publishConfig(); + + await waitFor(() => expect(modelCalls().length).toBeGreaterThan(0)); + }); + + it("stops calling the unpaginated public model hub route", async () => { + const networkingModule = await import("./networking"); + renderHub(); + + await waitFor(() => expect(apiGetMock).toHaveBeenCalled()); + expect(networkingModule.modelHubPublicModelsCall).not.toHaveBeenCalled(); + }); + + it("counts the whole catalogue from the response meta, not the rows on screen", async () => { + respondWith(DEFAULT_MODELS, 300); + renderHub(); + + await screen.findByText("gpt-4"); + expect(screen.getByTestId("pagination-range")).toHaveTextContent("of 300"); + expect(screen.getByTestId("pagination-page")).toHaveTextContent("Page 1 of 6"); + }); + + it("asks the server for the next page", async () => { + const user = userEvent.setup(); + respondWith(DEFAULT_MODELS, 300); + renderHub(); + await screen.findByText("gpt-4"); + + await user.click(screen.getByTestId("pagination-next")); + + await waitFor(() => expect(lastModelQuery().page).toBe(2)); + expect(lastModelQuery().page_size).toBe(50); + }); + + it("asks the server for a different page size", async () => { + const user = userEvent.setup(); + respondWith(DEFAULT_MODELS, 300); + renderHub(); + await screen.findByText("gpt-4"); + + await user.click(screen.getByTestId("pagination-page-size")); + await user.click(await screen.findByRole("option", { name: "25" })); + + await waitFor(() => expect(lastModelQuery().page_size).toBe(25)); + }); + + it("asks the server to sort, in the sort form the endpoint accepts", async () => { + const user = userEvent.setup(); + renderHub(); + await screen.findByText("gpt-4"); + + await user.click(screen.getByTestId("sort-header-model_group")); + await waitFor(() => expect(lastModelQuery().sort).toBe("-model_group")); + + await user.click(screen.getByTestId("sort-header-input_cost_per_token")); + await waitFor(() => expect(lastModelQuery().sort).toBe("-input_cost_per_token")); + }); + + it("renders the page in the order the server sent it, without re-sorting locally", async () => { + const user = userEvent.setup(); + respondWith([model({ model_group: "alpha-model" }), model({ model_group: "zeta-model" })], 300); + renderHub(); + await screen.findByText("alpha-model"); + + await user.click(screen.getByTestId("sort-header-model_group")); + await waitFor(() => expect(lastModelQuery().sort).toBe("-model_group")); + + const rendered = screen.getAllByText(/-model$/).map((cell) => cell.textContent); + expect(rendered).toEqual(["alpha-model", "zeta-model"]); + }); + + it("offers sorting on exactly the fields the endpoint accepts", async () => { + renderHub(); + await screen.findByText("gpt-4"); + + const sortable = screen + .getAllByTestId(/^sort-header-/) + .map((header) => header.getAttribute("data-testid")?.replace("sort-header-", "")); + + expect(sortable.sort()).toEqual([ + "input_cost_per_token", + "max_input_tokens", + "max_output_tokens", + "mode", + "model_group", + "output_cost_per_token", + "providers", + "rpm", + ]); + expect(screen.getByText("Health Status")).toBeInTheDocument(); + expect(screen.queryByTestId("sort-header-health_status")).not.toBeInTheDocument(); + }); + + it("searches on the server and returns to the first page", async () => { + const user = userEvent.setup(); + respondWith(DEFAULT_MODELS, 300); + renderHub(); + await screen.findByText("gpt-4"); + + await user.click(screen.getByTestId("pagination-next")); + await waitFor(() => expect(lastModelQuery().page).toBe(2)); + + await user.type(screen.getByPlaceholderText("Search model names..."), "claude"); + + await waitFor(() => expect(lastModelQuery().q).toBe("claude")); + expect(lastModelQuery().page).toBe(1); + }); + + it("filters by mode with the endpoint's in operator", async () => { + const user = userEvent.setup(); + renderHub(); + await screen.findByText("gpt-4"); + + await user.click(screen.getByPlaceholderText("Select modes")); + await user.click(await screen.findByRole("option", { name: "embedding" })); + + await waitFor(() => expect(lastModelQuery()["filter[mode][in]"]).toBe("embedding")); + }); + + it("filters by several providers at once, and returns to the first page", async () => { + const user = userEvent.setup(); + respondWith(DEFAULT_MODELS, 300); + renderHub(); + await screen.findByText("gpt-4"); + + await user.click(screen.getByTestId("pagination-next")); + await waitFor(() => expect(lastModelQuery().page).toBe(2)); + + await user.click(screen.getByPlaceholderText("Select providers")); + await user.click(await screen.findByRole("option", { name: /anthropic/i })); + await waitFor(() => expect(lastModelQuery()["filter[providers][in]"]).toBe("anthropic")); + expect(lastModelQuery().page).toBe(1); + + await user.click(await screen.findByRole("option", { name: /openai/i })); + + await waitFor(() => expect(lastModelQuery()["filter[providers][in]"]).toBe("anthropic,openai")); + }); + + it("filters by feature, which the table could not do while it paged", async () => { + const user = userEvent.setup(); + renderHub(); + await screen.findByText("gpt-4"); + + await user.click(screen.getByPlaceholderText("Select features")); + await user.click(await screen.findByRole("option", { name: "Vision" })); + + await waitFor(() => expect(lastModelQuery()["filter[features][in]"]).toBe("vision")); + }); + + it("offers the filter values the route reports, not the ones on the page", async () => { + respondWith([model({ model_group: "gpt-4" })], 1); + renderHub(); + await screen.findByText("gpt-4"); + + await waitFor(() => expect(facetPaths()).toContain(`${MODEL_HUB_PATH}/providers`)); + expect(facetPaths()).toEqual(expect.arrayContaining([`${MODEL_HUB_PATH}/modes`, `${MODEL_HUB_PATH}/features`])); + }); + it("displays health status correctly for models with health check information", async () => { - const mockModelsWithHealthChecks = [ + respondWith([ { + ...MODEL_DEFAULTS, model_group: "gpt-4", - providers: ["openai"], - mode: "chat", health_status: "healthy", health_response_time: 150.5, health_checked_at: "2024-01-15T10:30:00Z", - supports_function_calling: true, - supports_vision: false, - supports_parallel_function_calling: false, }, { + ...MODEL_DEFAULTS, model_group: "claude-3", providers: ["anthropic"], - mode: "chat", health_status: "unhealthy", health_response_time: 5000.0, health_checked_at: "2024-01-15T10:25:00Z", - supports_function_calling: true, - supports_vision: false, - supports_parallel_function_calling: false, }, - { - model_group: "gpt-3.5-turbo", - providers: ["openai"], - mode: "chat", - health_status: undefined, - health_response_time: undefined, - health_checked_at: undefined, - supports_function_calling: false, - supports_vision: false, - supports_parallel_function_calling: false, - }, - ]; + model({ model_group: "gpt-3.5-turbo" }), + ]); - const networkingModule = await import("./networking"); - vi.mocked(networkingModule.modelHubPublicModelsCall).mockResolvedValue(mockModelsWithHealthChecks); + renderHub(); - render(); - - // Wait for the component to load and render the table await waitFor(() => { expect(screen.getByText("gpt-4")).toBeInTheDocument(); }); - // Check the health status badge in each model's row await waitFor(() => { const gpt4Row = screen.getByText("gpt-4").closest("tr"); expect(gpt4Row).toBeInTheDocument(); @@ -134,19 +360,13 @@ describe("PublicModelHub", () => { expect(within(gpt35Row as HTMLElement).getByText("Unknown")).toBeInTheDocument(); }); }); - it("shows no models when the search has no matches (LIT-5230 regression)", async () => { - const networkingModule = await import("./networking"); - vi.mocked(networkingModule.modelHubPublicModelsCall).mockResolvedValue([ - { model_group: "gpt-4", providers: ["openai"], mode: "chat" }, - { model_group: "claude-3", providers: ["anthropic"], mode: "chat" }, - ]); - render(); + it("shows no models when the search has no matches (LIT-5230 regression)", async () => { + renderHub(); expect(await screen.findByText("gpt-4")).toBeInTheDocument(); - fireEvent.change(screen.getByPlaceholderText("Search model names... (smart search enabled)"), { - target: { value: "zzzz" }, - }); + respondWith([], 0); + fireEvent.change(screen.getByPlaceholderText("Search model names..."), { target: { value: "zzzz" } }); await waitFor(() => { expect(screen.queryByText("gpt-4")).not.toBeInTheDocument(); @@ -155,18 +375,23 @@ describe("PublicModelHub", () => { }); }); - it("handles non-array response gracefully (regression test for e.filter crash)", async () => { - const networkingModule = await import("./networking"); - // Mock the API to return an object (like an error response) instead of an array - vi.mocked(networkingModule.modelHubPublicModelsCall).mockResolvedValue({ - detail: "No models configured", - } as any); + it("reports the proxy as unavailable when the model page fails to load", async () => { + apiGetMock.mockRejectedValue(new Error("boom")); - render(); + renderHub(); + + expect(await screen.findByText(/Service unavailable/)).toBeInTheDocument(); + }); + + it("keeps the page usable when the response carries no rows", async () => { + respondWith([], 0); + + renderHub(); await waitFor(() => { expect(screen.getByTestId("navbar")).toBeInTheDocument(); expect(screen.getByText("Model Hub")).toBeInTheDocument(); + expect(screen.getByText("No models available")).toBeInTheDocument(); }); }); }); @@ -237,7 +462,7 @@ describe("public hub MCP details modal", () => { const networkingModule = await import("./networking"); vi.mocked(networkingModule.mcpHubPublicServersCall).mockResolvedValue([mockMcpServer]); - render(); + renderHub(); fireEvent.click(await screen.findByRole("tab", { name: /MCP Hub/i })); fireEvent.click(await screen.findByRole("button", { name: "exa_test" })); @@ -252,7 +477,7 @@ describe("public hub MCP details modal", () => { const networkingModule = await import("./networking"); vi.mocked(networkingModule.mcpHubPublicServersCall).mockResolvedValue([mockMcpServer]); - render(); + renderHub(); fireEvent.click(await screen.findByRole("tab", { name: /MCP Hub/i })); fireEvent.click(await screen.findByRole("button", { name: "exa_test" })); diff --git a/ui/litellm-dashboard/src/components/public_model_hub.tsx b/ui/litellm-dashboard/src/components/public_model_hub.tsx index f6364b5d9d1..8bd47e47a84 100644 --- a/ui/litellm-dashboard/src/components/public_model_hub.tsx +++ b/ui/litellm-dashboard/src/components/public_model_hub.tsx @@ -21,6 +21,9 @@ import { Dialog, DialogContent, DialogHeader, DialogTitle } from "@/components/u import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip"; import { MultiSelect } from "./shared/MultiSelect"; +import { featureLabel } from "./publicModelHub/publicModelHubFilters"; +import { usePublicModelHubFacets } from "./publicModelHub/usePublicModelHubFacets"; +import { usePublicModelHubList } from "./publicModelHub/usePublicModelHubList"; import { DataTable } from "./shared/DataTable"; import { toast } from "@/lib/toast"; import Navbar from "./navbar"; @@ -31,7 +34,6 @@ import { getPublicModelHubInfo, getUiConfig, mcpHubPublicServersCall, - modelHubPublicModelsCall, } from "./networking"; import { Plugin } from "./claude_code_plugins/types"; import SkillHubDashboard from "./AIHub/SkillHubDashboard"; @@ -68,25 +70,19 @@ function PublicHubEmptyState({ title, body }: { title: string; body: string }) { const PublicModelHub: React.FC = ({ accessToken, isEmbedded = false }) => { const anchor = useComboboxAnchor(); - const [modelHubData, setModelHubData] = useState(null); + const [proxyConfigured, setProxyConfigured] = useState(false); const [agentHubData, setAgentHubData] = useState(null); const [mcpHubData, setMcpHubData] = useState(null); const [pageTitle, setPageTitle] = useState("LiteLLM Gateway"); const [customDocsDescription, setCustomDocsDescription] = useState(null); const [litellmVersion, setLitellmVersion] = useState(""); const [usefulLinks, setUsefulLinks] = useState>({}); - const [loading, setLoading] = useState(true); const [agentLoading, setAgentLoading] = useState(true); const [mcpLoading, setMcpLoading] = useState(true); - const [searchTerm, setSearchTerm] = useState(""); const [agentSearchTerm, setAgentSearchTerm] = useState(""); const [mcpSearchTerm, setMcpSearchTerm] = useState(""); - const [selectedProviders, setSelectedProviders] = useState([]); - const [selectedModes, setSelectedModes] = useState([]); - const [selectedFeatures, setSelectedFeatures] = useState([]); const [selectedAgentSkills, setSelectedAgentSkills] = useState([]); const [selectedMcpTransports, setSelectedMcpTransports] = useState([]); - const [serviceStatus, setServiceStatus] = useState("I'm alive! ✓"); const [isModalVisible, setIsModalVisible] = useState(false); const [isAgentModalVisible, setIsAgentModalVisible] = useState(false); const [isMcpModalVisible, setIsMcpModalVisible] = useState(false); @@ -106,19 +102,7 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded console.error("Failed to get UI config:", error); // Continue anyway - might work with default proxyBaseUrl } - - const fetchPublicData = async () => { - try { - setLoading(true); - const _modelHubData = await modelHubPublicModelsCall(); - setModelHubData(Array.isArray(_modelHubData) ? _modelHubData : []); - } catch (error) { - console.error("There was an error fetching the public model data", error); - setServiceStatus("Service unavailable"); - } finally { - setLoading(false); - } - }; + setProxyConfigured(true); const fetchAgentData = async () => { try { @@ -166,7 +150,6 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded fetchPublicModelHubInfo(); - fetchPublicData(); fetchAgentData(); fetchMcpData(); fetchSkillData(); @@ -175,47 +158,6 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded initializeAndFetch(); }, []); - // Clear filters when filter values change to avoid confusion - useEffect(() => { - // This would clear selections if we had any selection functionality - // For now, it's just for consistency with the original component - }, [searchTerm, selectedProviders, selectedModes, selectedFeatures]); - - const getUniqueProviders = (data: ModelGroupInfo[]) => { - const providers = new Set(); - data.forEach((model) => { - (model.providers ?? []).forEach((provider) => providers.add(provider)); - }); - return Array.from(providers); - }; - - const getUniqueModes = (data: ModelGroupInfo[]) => { - const modes = new Set(); - data.forEach((model) => { - if (model.mode) modes.add(model.mode); - }); - return Array.from(modes); - }; - - const getUniqueFeatures = (data: ModelGroupInfo[]) => { - const features = new Set(); - data.forEach((model) => { - // Find all properties that start with 'supports_' and are true - Object.entries(model) - .filter(([key, value]) => key.startsWith("supports_") && value === true) - .forEach(([key]) => { - // Format the feature name (remove 'supports_' prefix and convert to title case) - const featureName = key - .replace(/^supports_/, "") - .split("_") - .map((word) => word.charAt(0).toUpperCase() + word.slice(1)) - .join(" "); - features.add(featureName); - }); - }); - return Array.from(features).sort(); - }; - const getUniqueAgentSkills = (data: AgentCard[]) => { const skills = new Set(); data.forEach((agent) => { @@ -234,39 +176,6 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded return Array.from(transports).sort(); }; - const filteredData = useMemo(() => { - if (!modelHubData || !Array.isArray(modelHubData)) return []; - - const searchResults = rankBySearchRelevance( - filterBySearchTerm(modelHubData, searchTerm, (model) => [model.model_group]), - searchTerm, - (model) => model.model_group, - ); - - // Apply other filters - return searchResults.filter((model) => { - const matchesProvider = - selectedProviders.length === 0 || selectedProviders.some((provider) => model.providers.includes(provider)); - const matchesMode = selectedModes.length === 0 || selectedModes.includes(model.mode || ""); - - // Check if model has any of the selected features - const matchesFeature = - selectedFeatures.length === 0 || - Object.entries(model) - .filter(([key, value]) => key.startsWith("supports_") && value === true) - .some(([key]) => { - const featureName = key - .replace(/^supports_/, "") - .split("_") - .map((word) => word.charAt(0).toUpperCase() + word.slice(1)) - .join(" "); - return selectedFeatures.includes(featureName); - }); - - return matchesProvider && matchesMode && matchesFeature; - }); - }, [modelHubData, searchTerm, selectedProviders, selectedModes, selectedFeatures]); - const filteredAgentData = useMemo(() => { if (!agentHubData || !Array.isArray(agentHubData)) return []; @@ -356,7 +265,14 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded return `$${(cost * 1_000_000).toFixed(4)}`; }; - const [modelSorting, setModelSorting] = useState([{ id: "model_group", desc: false }]); + const models = usePublicModelHubList(proxyConfigured); + const modelFacets = usePublicModelHubFacets(proxyConfigured); + const modeOptions = useMemo(() => modelFacets.modes.map((mode) => ({ label: mode, value: mode })), [modelFacets]); + const featureOptions = useMemo( + () => modelFacets.features.map((feature) => ({ label: featureLabel(feature), value: feature })), + [modelFacets], + ); + const serviceStatus = models.error ? "Service unavailable" : "I'm alive! ✓"; const [agentSorting, setAgentSorting] = useState([{ id: "name", desc: false }]); const [mcpSorting, setMcpSorting] = useState([{ id: "server_name", desc: false }]); @@ -367,22 +283,6 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded const hasAgents = Array.isArray(agentHubData) && agentHubData.length > 0; const hasMcpServers = Array.isArray(mcpHubData) && mcpHubData.length > 0; - const providerOptions = useMemo( - () => (Array.isArray(modelHubData) ? getUniqueProviders(modelHubData) : []), - [modelHubData], - ); - const modeOptions = useMemo( - () => - Array.isArray(modelHubData) ? getUniqueModes(modelHubData).map((mode) => ({ label: mode, value: mode })) : [], - [modelHubData], - ); - const featureOptions = useMemo( - () => - Array.isArray(modelHubData) - ? getUniqueFeatures(modelHubData).map((feature) => ({ label: feature, value: feature })) - : [], - [modelHubData], - ); const agentSkillOptions = useMemo( () => Array.isArray(agentHubData) @@ -495,9 +395,8 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded } /> - Smart search with relevance ranking - finds models containing your search terms, ranked by - relevance. Try searching 'xai grok-4', 'claude-4', 'gpt-4', or - 'sonnet' + Finds every published model whose name contains what you type, across all pages. Try + 'grok', 'claude', 'gpt-4', or 'sonnet' @@ -505,9 +404,10 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded setSearchTerm(e.target.value)} + placeholder="Search model names..." + aria-label="Search model names" + value={models.searchValue} + onChange={(e) => models.onSearchChange(e.target.value)} className="border border-border rounded-lg pl-10 pr-4 py-2 w-full text-sm focus:outline-hidden focus:ring-2 focus:ring-ring focus:border-transparent bg-card" /> @@ -516,9 +416,9 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded

Provider:

setSelectedProviders(values)} + items={modelFacets.providers} + value={models.providerValues} + onValueChange={models.onProvidersChange} > } className="min-h-8 w-full py-1 text-sm"> @@ -567,8 +467,8 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded

Mode:

@@ -577,8 +477,8 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded

Features:

@@ -586,19 +486,23 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded model.model_group || String(index)} - sortingMode="client" - sorting={modelSorting} - onSortingChange={setModelSorting} - isLoading={loading} + sortingMode="server" + sorting={models.sorting} + onSortingChange={models.onSortingChange} + paginationMode="server" + pagination={models.pagination} + onPaginationChange={models.onPaginationChange} + rowCount={models.rowCount} + isLoading={models.isLoading} loadingMessage="Loading models…" noDataMessage={ = ({ accessToken, isEmbedded } size="compact" /> - -
-

- Showing {filteredData.length} of {modelHubData?.length || 0} models -

-
{/* Agents Tab */} diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 3dcfeb64866..6942bb60566 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -12382,6 +12382,36 @@ export interface paths { patch?: never; trace?: never; }; + "/public/v1/model_hub/{facet}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Public Model Hub Facet + * @description The distinct providers, modes or features across the published model groups, for the + * Model Hub's filter dropdowns. No authentication. + * + * Carries the same filters and search as the list route, so a dropdown offers exactly + * the values the table can show: asking for providers under `filter[mode][in]=chat` + * lists only the providers that serve a chat model. + * + * Example curl: + * ``` + * curl --location --globoff 'http://0.0.0.0:4000/public/v1/model_hub/providers?filter[mode][in]=chat&page_size=50' + * ``` + */ + get: operations["public_model_hub_facet_public_v1_model_hub__facet__get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/queue/chat/completions": { parameters: { query?: never; @@ -55040,6 +55070,37 @@ export interface operations { }; }; }; + public_model_hub_facet_public_v1_model_hub__facet__get: { + parameters: { + query?: never; + header?: never; + path: { + facet: "providers" | "modes" | "features"; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["FacetListResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; async_queue_request_queue_chat_completions_post: { parameters: { query?: {