diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index c23678c51ae..ed9d8800202 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -141,6 +141,7 @@ jobs: test-path: >- tests/test_litellm/proxy/analytics_endpoints tests/test_litellm/proxy/management_endpoints + tests/test_litellm/proxy/list_api tests/test_litellm/proxy/memory tests/test_litellm/proxy/guardrails tests/test_litellm/proxy/management_helpers diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index ac98939288e..516fb620db6 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -730,6 +730,7 @@ class LiteLLMRoutes(enum.Enum): "/litellm/.well-known/litellm-ui-config", "/.well-known/litellm-ui-config", "/public/model_hub", + "/public/v1/model_hub", "/public/model_hub/info", "/public/agent_hub", "/public/mcp_hub", diff --git a/litellm/proxy/list_api/__init__.py b/litellm/proxy/list_api/__init__.py new file mode 100644 index 00000000000..919cb7d8bde --- /dev/null +++ b/litellm/proxy/list_api/__init__.py @@ -0,0 +1 @@ +"""Surface-neutral machinery for LiteLLM's own paginated list endpoints.""" diff --git a/litellm/proxy/list_api/common.py b/litellm/proxy/list_api/common.py new file mode 100644 index 00000000000..7ef2827f30e --- /dev/null +++ b/litellm/proxy/list_api/common.py @@ -0,0 +1,104 @@ +"""Contract machinery shared by every LiteLLM-defined list route, on any surface.""" + +from typing import Final +from urllib.parse import urlencode + +from fastapi import Request +from fastapi.dependencies.utils import get_flat_params +from fastapi.params import ParamTypes +from fastapi.responses import JSONResponse + +from litellm.types.proxy.management_endpoints.management_v1 import ( + ListLinks, + PageLinks, + ProblemDetail, +) + +PROBLEM_CONTENT_TYPE: Final = "application/problem+json" +# A URN, not an https URL: RFC 9457 only asks that `type` identify the problem +# type, and an https URI promises documentation at that address. Switch to an +# https base only when pages actually exist to serve. +PROBLEM_TYPE_BASE: Final = "urn:litellm:error:" + + +class ManagementProblem(Exception): + """Raised to return an RFC 9457 problem instead of the proxy's OpenAI error shape.""" + + def __init__(self, problem: ProblemDetail) -> None: + self.problem = problem + super().__init__(problem.detail) + + +def problem_response(problem: ProblemDetail) -> JSONResponse: + return JSONResponse( + status_code=problem.status, + content=problem.model_dump(exclude_none=True), + media_type=PROBLEM_CONTENT_TYPE, + ) + + +def _declared_query_params(request: Request) -> frozenset[str]: + route: Final = request.scope.get("route") + dependant: Final = getattr(route, "dependant", None) + if dependant is None: + return frozenset() + # fastapi>=0.140.7 removed get_flat_dependant(); get_flat_params() returns the + # flattened (deduped) param list. Filter to query params to match the old behavior. + return frozenset( + field.alias + for field in get_flat_params(dependant) + if getattr(field.field_info, "in_", None) == ParamTypes.query + ) + + +def escape_like(value: str) -> str: + """Escape LIKE/ILIKE metacharacters. Ids routinely contain `_`, which is a wildcard unescaped.""" + return value.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + + +def unknown_query_param_problem(unknown: tuple[str, ...], allowed: tuple[str, ...]) -> ProblemDetail: + return ProblemDetail( + type=f"{PROBLEM_TYPE_BASE}unknown-query-parameter", + title="Unknown query parameter", + status=400, + detail=f"Unrecognized query parameter(s): {', '.join(unknown)}.", + allowed=sorted(allowed), + ) + + +async def reject_unknown_query_params(request: Request) -> None: + """Reject any query param the route did not declare. + + A silently ignored filter over-returns data, which is worse than a rejected + request; a fresh surface is the only chance to be strict about it. + """ + declared: Final = _declared_query_params(request) + unknown: Final[tuple[str, ...]] = tuple(sorted(name for name in request.query_params if name not in declared)) + if not unknown: + return + raise ManagementProblem(unknown_query_param_problem(unknown=unknown, allowed=tuple(sorted(declared)))) + + +def _page_url(request: Request, page: int) -> str: + others: Final = tuple((key, value) for key, value in request.query_params.multi_items() if key != "page") + return f"{request.url.path}?{urlencode((*others, ('page', page)))}" + + +def build_page_links(request: Request, page: int, has_more: bool) -> PageLinks: + return PageLinks( + self_link=_page_url(request, page), + prev=_page_url(request, page - 1) if page > 1 else None, + next=_page_url(request, page + 1) if has_more else None, + ) + + +def build_list_links(request: Request, page: int, total_pages: int) -> ListLinks: + """Page-mode links. `last` clamps to page 1 on an empty result set so every link still resolves.""" + last: Final = max(total_pages, 1) + return ListLinks( + self_link=_page_url(request, page), + first=_page_url(request, 1), + prev=_page_url(request, page - 1) if page > 1 else None, + next=_page_url(request, page + 1) if page < last else None, + last=_page_url(request, last), + ) diff --git a/litellm/proxy/list_api/in_memory.py b/litellm/proxy/list_api/in_memory.py new file mode 100644 index 00000000000..bada8ea0a35 --- /dev/null +++ b/litellm/proxy/list_api/in_memory.py @@ -0,0 +1,143 @@ +"""An in-memory `ListExecutor`, for list resources whose rows are computed rather than queried. + +Answers the same `QueryPlan` a SQL executor would render through `where_sql` / `order_by_sql`, +so a filter or a sort means the same thing on either. `enrich_page` runs on the page slice and +never on the whole match set. +""" + +from collections.abc import Awaitable, Callable, Mapping, Sequence +from dataclasses import dataclass +from datetime import datetime +from functools import reduce +from typing import Final, Generic, TypeAlias, TypeVar + +from typing_extensions import assert_never + +from litellm.proxy.list_api.list_framework import ( + AnyOf, + Compare, + ComparisonOp, + FilterValue, + IsNull, + Predicate, + QueryPlan, + SortKey, + Within, +) + +TRow: Final = TypeVar("TRow") + +Cell: TypeAlias = str | int | float | datetime | None +# A tuple-valued cell is a row's repeated field (a model group's providers, say). A predicate +# holds against it when it holds against any one element, the way an SQL join would answer. +Cells: TypeAlias = Mapping[str, Cell | tuple[Cell, ...]] + + +def _sign(cell: Cell, value: FilterValue) -> int | None: + """None when the two values are not orderable against each other.""" + if isinstance(cell, str) and isinstance(value, str): + return (cell > value) - (cell < value) + if isinstance(cell, datetime) and isinstance(value, datetime): + return (cell > value) - (cell < value) + if isinstance(cell, (int, float)) and isinstance(value, (int, float)): + return (cell > value) - (cell < value) + return None + + +def _matches(cell: Cell, op: ComparisonOp, value: FilterValue) -> bool: + """SQL's three-valued logic: a NULL cell satisfies no comparison, only `is_null`.""" + if cell is None: + return False + sign: Final = _sign(cell, value) + match op: + case "eq": + return cell == value + case "not": + return cell != value + case "contains": + return str(value).casefold() in str(cell).casefold() + case "gt": + return sign is not None and sign > 0 + case "gte": + return sign is not None and sign >= 0 + case "lt": + return sign is not None and sign < 0 + case "lte": + return sign is not None and sign <= 0 + case _: + assert_never(op) + + +def _any_cell(cells: Cells, name: str, matches: Callable[[Cell], bool]) -> bool: + cell: Final = cells.get(name) + if isinstance(cell, tuple): + return any(matches(item) for item in cell) + return matches(cell) + + +def _leaf_holds(predicate: Compare | Within | IsNull, cells: Cells) -> bool: + match predicate: + case Compare(field=name, op=op, value=value): + return _any_cell(cells, name, lambda cell: _matches(cell, op, value)) + case Within(field=name, values=values): + return _any_cell(cells, name, lambda cell: cell is not None and cell in values) + case IsNull(field=name, negated=negated): + return _any_cell(cells, name, lambda cell: (cell is None) != negated) + case _: + assert_never(predicate) + + +def _holds(predicate: Predicate, cells: Cells) -> bool: + if isinstance(predicate, AnyOf): + return any(_leaf_holds(clause, cells) for clause in predicate.clauses) + return _leaf_holds(predicate, cells) + + +def _sort_key(cells: Cells, key: SortKey) -> tuple[bool, Cell | tuple[Cell, ...]]: + """NULLS LAST in both directions, matching `order_by_sql`. + + The placeholder standing in for a null is only ever compared against another null's, + because the rank ahead of it already separates nulls from the rest. + """ + cell: Final = cells.get(key.field) + return (cell is None) != key.descending, 0 if cell is None else cell + + +def _ordered( + matched: Sequence[tuple[Cells, TRow]], + order: tuple[SortKey, ...], +) -> Sequence[tuple[Cells, TRow]]: + """Least significant key first: Python's sort is stable, so the most significant pass wins.""" + return reduce( + lambda rows, key: sorted(rows, key=lambda pair: _sort_key(pair[0], key), reverse=key.descending), + reversed(order), + matched, + ) + + +async def _unchanged(rows: Sequence[TRow]) -> Sequence[TRow]: + return rows + + +@dataclass(frozen=True, slots=True) +class InMemoryListExecutor(Generic[TRow]): + """`cells` projects a row down to the values the spec's filters, search and sort read, so a + plan can be applied without this module knowing the row type.""" + + rows: Sequence[TRow] + cells: Callable[[TRow], Cells] + enrich_page: Callable[[Sequence[TRow]], Awaitable[Sequence[TRow]]] = _unchanged + + def _matching(self, where: tuple[Predicate, ...]) -> Sequence[tuple[Cells, TRow]]: + return tuple( + (cells, row) + for cells, row in ((self.cells(row), row) for row in self.rows) + if all(_holds(predicate, cells) for predicate in where) + ) + + async def count(self, where: tuple[Predicate, ...]) -> int: + return len(self._matching(where)) + + async def find_many(self, plan: QueryPlan) -> Sequence[TRow]: + page: Final = _ordered(self._matching(plan.where), plan.order)[plan.skip : plan.skip + plan.take] + return await self.enrich_page(tuple(row for _, row in page)) diff --git a/litellm/proxy/management_endpoints/management_v1/list_framework.py b/litellm/proxy/list_api/list_framework.py similarity index 94% rename from litellm/proxy/management_endpoints/management_v1/list_framework.py rename to litellm/proxy/list_api/list_framework.py index fd366e81934..21ee4e6860f 100644 --- a/litellm/proxy/management_endpoints/management_v1/list_framework.py +++ b/litellm/proxy/list_api/list_framework.py @@ -1,4 +1,4 @@ -"""Generic list handling for `/management/v1` collection routes. +"""Generic list handling for LiteLLM-defined collection routes. A resource declares a `ListSpec`; `build_query_plan` turns query parameters into a `QueryPlan` or an RFC 9457 problem without touching a database, and `handle_list` @@ -24,7 +24,7 @@ from pydantic import TypeAdapter, ValidationError from typing_extensions import assert_never from litellm.proxy._types import UserAPIKeyAuth -from litellm.proxy.management_endpoints.management_v1.common import ( +from litellm.proxy.list_api.common import ( PROBLEM_TYPE_BASE, ManagementProblem, build_list_links, @@ -85,9 +85,13 @@ class IsNull: @dataclass(frozen=True, slots=True) class AnyOf: - """Disjunction of its clauses. `?q=` is the only producer today.""" + """Disjunction of its clauses. `?q=` is the only producer. - clauses: tuple["Predicate", ...] + Holding leaves rather than predicates keeps the disjunction one level deep by type, so + neither the SQL renderer nor an in-memory executor has to walk a tree to evaluate it. + """ + + clauses: tuple[Compare, ...] Predicate = Compare | Within | IsNull | AnyOf @@ -369,6 +373,19 @@ def _parse_sort(spec: ListSpec[TRow, TOut], params: Mapping[str, str]) -> tuple[ f"Cannot sort {spec.resource} by: {', '.join(repr(field) for field in rejected)}.", tuple(spec.sortable), ) + # A repeated field cannot change the ordering, but an executor that sorts once per key + # does the work anyway. Rejecting repeats bounds that to the size of `sortable`, which + # matters because an unauthenticated caller can otherwise name one field a thousand times. + fields: Final = tuple(key.field for key in keys) + repeated: Final = tuple(sorted(frozenset(field for field in fields if fields.count(field) > 1))) + if repeated: + return _problem( + "duplicate-sort-field", + "Duplicate sort field", + 400, + f"Sort field(s) named more than once: {', '.join(repeated)}. Each may appear once.", + tuple(spec.sortable), + ) return keys diff --git a/litellm/proxy/management_endpoints/management_v1/budgets.py b/litellm/proxy/management_endpoints/management_v1/budgets.py index 51ebc20fe31..cc2fefc426f 100644 --- a/litellm/proxy/management_endpoints/management_v1/budgets.py +++ b/litellm/proxy/management_endpoints/management_v1/budgets.py @@ -16,12 +16,11 @@ from litellm.proxy._types import ( user_api_key_has_admin_view, ) from litellm.proxy.auth.user_api_key_auth import user_api_key_auth -from litellm.proxy.management_endpoints.management_v1.common import ( - MANAGEMENT_V1_PREFIX, +from litellm.proxy.list_api.common import ( PROBLEM_TYPE_BASE, ManagementProblem, ) -from litellm.proxy.management_endpoints.management_v1.list_framework import ( +from litellm.proxy.list_api.list_framework import ( FilterSpec, ListSpec, Predicate, @@ -34,6 +33,7 @@ from litellm.proxy.management_endpoints.management_v1.list_framework import ( order_by_sql, where_sql, ) +from litellm.proxy.management_endpoints.management_v1.common import MANAGEMENT_V1_PREFIX from litellm.proxy.utils import PrismaClient from litellm.types.proxy.management_endpoints.management_v1 import ( ListResponse, diff --git a/litellm/proxy/management_endpoints/management_v1/common.py b/litellm/proxy/management_endpoints/management_v1/common.py index ec79820465a..5ecaacbe170 100644 --- a/litellm/proxy/management_endpoints/management_v1/common.py +++ b/litellm/proxy/management_endpoints/management_v1/common.py @@ -1,105 +1,8 @@ -"""Contract machinery shared by every `/management/v1` route.""" +"""Constants specific to the `/management/v1` control-plane surface. + +The contract machinery every list route shares lives in `litellm.proxy.list_api`. +""" from typing import Final -from urllib.parse import urlencode - -from fastapi import Request -from fastapi.dependencies.utils import get_flat_params -from fastapi.params import ParamTypes -from fastapi.responses import JSONResponse - -from litellm.types.proxy.management_endpoints.management_v1 import ( - ListLinks, - PageLinks, - ProblemDetail, -) MANAGEMENT_V1_PREFIX: Final = "/management/v1" -PROBLEM_CONTENT_TYPE: Final = "application/problem+json" -# A URN, not an https URL: RFC 9457 only asks that `type` identify the problem -# type, and an https URI promises documentation at that address. Switch to an -# https base only when pages actually exist to serve. -PROBLEM_TYPE_BASE: Final = "urn:litellm:error:" - - -class ManagementProblem(Exception): - """Raised to return an RFC 9457 problem instead of the proxy's OpenAI error shape.""" - - def __init__(self, problem: ProblemDetail) -> None: - self.problem = problem - super().__init__(problem.detail) - - -def problem_response(problem: ProblemDetail) -> JSONResponse: - return JSONResponse( - status_code=problem.status, - content=problem.model_dump(exclude_none=True), - media_type=PROBLEM_CONTENT_TYPE, - ) - - -def _declared_query_params(request: Request) -> frozenset[str]: - route: Final = request.scope.get("route") - dependant: Final = getattr(route, "dependant", None) - if dependant is None: - return frozenset() - # fastapi>=0.140.7 removed get_flat_dependant(); get_flat_params() returns the - # flattened (deduped) param list. Filter to query params to match the old behavior. - return frozenset( - field.alias - for field in get_flat_params(dependant) - if getattr(field.field_info, "in_", None) == ParamTypes.query - ) - - -def escape_like(value: str) -> str: - """Escape LIKE/ILIKE metacharacters. Ids routinely contain `_`, which is a wildcard unescaped.""" - return value.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") - - -def unknown_query_param_problem(unknown: tuple[str, ...], allowed: tuple[str, ...]) -> ProblemDetail: - return ProblemDetail( - type=f"{PROBLEM_TYPE_BASE}unknown-query-parameter", - title="Unknown query parameter", - status=400, - detail=f"Unrecognized query parameter(s): {', '.join(unknown)}.", - allowed=sorted(allowed), - ) - - -async def reject_unknown_query_params(request: Request) -> None: - """Reject any query param the route did not declare. - - A silently ignored filter over-returns data, which is worse than a rejected - request; a fresh surface is the only chance to be strict about it. - """ - declared: Final = _declared_query_params(request) - unknown: Final[tuple[str, ...]] = tuple(sorted(name for name in request.query_params if name not in declared)) - if not unknown: - return - raise ManagementProblem(unknown_query_param_problem(unknown=unknown, allowed=tuple(sorted(declared)))) - - -def _page_url(request: Request, page: int) -> str: - others: Final = tuple((key, value) for key, value in request.query_params.multi_items() if key != "page") - return f"{request.url.path}?{urlencode((*others, ('page', page)))}" - - -def build_page_links(request: Request, page: int, has_more: bool) -> PageLinks: - return PageLinks( - self_link=_page_url(request, page), - prev=_page_url(request, page - 1) if page > 1 else None, - next=_page_url(request, page + 1) if has_more else None, - ) - - -def build_list_links(request: Request, page: int, total_pages: int) -> ListLinks: - """Page-mode links. `last` clamps to page 1 on an empty result set so every link still resolves.""" - last: Final = max(total_pages, 1) - return ListLinks( - self_link=_page_url(request, page), - first=_page_url(request, 1), - prev=_page_url(request, page - 1) if page > 1 else None, - next=_page_url(request, page + 1) if page < last else None, - last=_page_url(request, last), - ) diff --git a/litellm/proxy/management_endpoints/management_v1/spend_logs.py b/litellm/proxy/management_endpoints/management_v1/spend_logs.py index 5fee8eaede3..f6907a7f87a 100644 --- a/litellm/proxy/management_endpoints/management_v1/spend_logs.py +++ b/litellm/proxy/management_endpoints/management_v1/spend_logs.py @@ -8,14 +8,14 @@ from fastapi import APIRouter, Depends, Query, Request from litellm._logging import verbose_proxy_logger from litellm.proxy._types import CommonProxyErrors, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth -from litellm.proxy.management_endpoints.management_v1.common import ( - MANAGEMENT_V1_PREFIX, +from litellm.proxy.list_api.common import ( PROBLEM_TYPE_BASE, ManagementProblem, build_page_links, escape_like, reject_unknown_query_params, ) +from litellm.proxy.management_endpoints.management_v1.common import MANAGEMENT_V1_PREFIX from litellm.proxy.utils import PrismaClient from litellm.types.proxy.management_endpoints.management_v1 import ( FacetListResponse, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 68d0960905f..597acdf661f 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -432,6 +432,11 @@ from litellm.proxy.hooks.prompt_injection_detection import ( ) from litellm.proxy.hooks.proxy_track_cost_callback import _ProxyDBLogger from litellm.proxy.image_endpoints.endpoints import router as image_router +from litellm.proxy.list_api.common import ( + PROBLEM_TYPE_BASE, + ManagementProblem, + problem_response, +) from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request from litellm.proxy.logging_endpoints.callback_logs_endpoints import ( rust_control_plane_router, @@ -488,12 +493,7 @@ from litellm.proxy.management_endpoints.key_management_endpoints import ( from litellm.proxy.management_endpoints.management_v1 import ( router as management_v1_router, ) -from litellm.proxy.management_endpoints.management_v1.common import ( - MANAGEMENT_V1_PREFIX, - PROBLEM_TYPE_BASE, - ManagementProblem, - problem_response, -) +from litellm.proxy.management_endpoints.management_v1.common import MANAGEMENT_V1_PREFIX from litellm.proxy.management_endpoints.model_access_group_management_endpoints import ( router as model_access_group_management_router, ) @@ -600,6 +600,7 @@ from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( router as pass_through_router, ) from litellm.proxy.public_endpoints import router as public_endpoints_router +from litellm.proxy.public_endpoints.public_v1 import router as public_v1_router from litellm.proxy.rag_endpoints.endpoints import router as rag_router from litellm.proxy.rerank_endpoints.endpoints import router as rerank_router from litellm.proxy.response_api_endpoints.endpoints import router as response_router @@ -17684,6 +17685,7 @@ async def get_routes(): app.include_router(router) app.include_router(response_router) app.include_router(public_endpoints_router) +app.include_router(public_v1_router) app.include_router(rerank_router) app.include_router(ocr_router) app.include_router(rag_router) diff --git a/litellm/proxy/public_endpoints/public_v1/__init__.py b/litellm/proxy/public_endpoints/public_v1/__init__.py new file mode 100644 index 00000000000..158bfdb3b66 --- /dev/null +++ b/litellm/proxy/public_endpoints/public_v1/__init__.py @@ -0,0 +1,14 @@ +"""The `/public/v1` unauthenticated public surface.""" + +from typing import Final + +from fastapi import APIRouter + +from litellm.proxy.public_endpoints.public_v1.model_hub import router as model_hub_router + +PUBLIC_V1_PREFIX: Final = "/public/v1" + +router: Final = APIRouter(prefix=PUBLIC_V1_PREFIX) +router.include_router(model_hub_router) + +__all__ = ("PUBLIC_V1_PREFIX", "router") diff --git a/litellm/proxy/public_endpoints/public_v1/model_hub.py b/litellm/proxy/public_endpoints/public_v1/model_hub.py new file mode 100644 index 00000000000..5a2d8068af7 --- /dev/null +++ b/litellm/proxy/public_endpoints/public_v1/model_hub.py @@ -0,0 +1,242 @@ +"""`GET /public/v1/model_hub`.""" + +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from types import MappingProxyType +from typing import Annotated, Final, Protocol + +from fastapi import APIRouter, Depends, Request +from typing_extensions import ReadOnly, TypedDict + +import litellm +from litellm._logging import verbose_proxy_logger +from litellm.proxy._types import CommonProxyErrors, UserAPIKeyAuth +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.list_api.common import PROBLEM_TYPE_BASE, ManagementProblem +from litellm.proxy.list_api.in_memory import Cells, InMemoryListExecutor +from litellm.proxy.list_api.list_framework import ( + FilterSpec, + ListSpec, + Scope, + ScopeAll, + SortKey, + handle_list, +) +from litellm.proxy.utils import PrismaClient +from litellm.types.proxy.management_endpoints.management_v1 import ( + ListResponse, + ProblemDetail, +) +from litellm.types.proxy.management_endpoints.model_management_endpoints import ( + ModelGroupInfoProxy, +) + +router: Final = APIRouter() + + +@dataclass(frozen=True, slots=True) +class HealthSnapshot: + """The health fields a model hub row carries, as the latest health check recorded them.""" + + status: str | None + response_time_ms: float | None + checked_at: str | None + + +class HealthSnapshotLookup(Protocol): + """The health half of the list, injected so the page slice decides how much of it runs.""" + + async def latest_for(self, model_groups: Sequence[str]) -> Mapping[str, HealthSnapshot]: ... + + +@dataclass(frozen=True, slots=True) +class PrismaHealthSnapshotLookup: + prisma_client: PrismaClient + + async def latest_for(self, model_groups: Sequence[str]) -> Mapping[str, HealthSnapshot]: + checks: Final = await self.prisma_client.get_latest_health_checks_for_models(model_groups) + return MappingProxyType( + { + check.model_name: HealthSnapshot( + status=check.status, + response_time_ms=check.response_time_ms, + checked_at=check.checked_at.isoformat() if check.checked_at else None, + ) + for check in checks + } + ) + + +class _HealthFields(TypedDict): + health_status: ReadOnly[str | None] + health_response_time: ReadOnly[float | None] + health_checked_at: ReadOnly[str | None] + + +def _with_health(row: ModelGroupInfoProxy, health: HealthSnapshot | None) -> ModelGroupInfoProxy: + if health is None: + return row + update: Final[_HealthFields] = { + "health_status": health.status, + "health_response_time": health.response_time_ms, + "health_checked_at": health.checked_at, + } + return row.model_copy(update=update) + + +@dataclass(frozen=True, slots=True) +class HealthEnricher: + """Resolves health for exactly the rows handed to it, which is the page and never the match set.""" + + lookup: HealthSnapshotLookup + + async def __call__(self, rows: Sequence[ModelGroupInfoProxy]) -> Sequence[ModelGroupInfoProxy]: + health: Final = await self.lookup.latest_for(tuple(row.model_group for row in rows)) + return tuple(_with_health(row, health.get(row.model_group)) for row in rows) + + +def _cells(row: ModelGroupInfoProxy) -> Cells: + return MappingProxyType( + { + "model_group": row.model_group, + "mode": row.mode, + "providers": tuple(row.providers), + "max_input_tokens": row.max_input_tokens, + "max_output_tokens": row.max_output_tokens, + "input_cost_per_token": row.input_cost_per_token, + "output_cost_per_token": row.output_cost_per_token, + } + ) + + +def _serialize(row: ModelGroupInfoProxy) -> ModelGroupInfoProxy: + """The row shape is the wire shape: the rows served are the router's own model group records.""" + return row + + +def _scope(_caller: UserAPIKeyAuth) -> Scope: + """Unconditional, and `/public/v1` is the one surface where that is allowed. + + Every row here is already a model group the operator published, so a public browse + caller seeing all of them is the answer, not a gap in the scoping. + """ + return ScopeAll() + + +MODEL_HUB_FILTERS: Final[Mapping[str, FilterSpec]] = MappingProxyType( + { + "mode": FilterSpec(type=str, ops=frozenset(("eq", "in"))), + "providers": FilterSpec(type=str, ops=frozenset(("contains",))), + } +) + +MODEL_HUB_LIST_SPEC: Final[ListSpec[ModelGroupInfoProxy, ModelGroupInfoProxy]] = ListSpec( + resource="model groups", + sortable=frozenset( + ( + "model_group", + "mode", + "max_input_tokens", + "max_output_tokens", + "input_cost_per_token", + "output_cost_per_token", + ) + ), + searchable=frozenset(("model_group",)), + filters=MODEL_HUB_FILTERS, + default_sort=(SortKey(field="model_group", descending=False),), + default_page_size=50, + max_page_size=100, + scope=_scope, + serialize=_serialize, + tiebreaker="model_group", +) + + +def _executor( + rows: Sequence[ModelGroupInfoProxy], + prisma_client: PrismaClient | None, +) -> InMemoryListExecutor[ModelGroupInfoProxy]: + if prisma_client is None: + return InMemoryListExecutor(rows=rows, cells=_cells) + return InMemoryListExecutor( + rows=rows, + cells=_cells, + enrich_page=HealthEnricher(lookup=PrismaHealthSnapshotLookup(prisma_client=prisma_client)), + ) + + +@router.get( + "/model_hub", + tags=["public", "model management"], # mutable-ok: fastapi types tags as list[str | Enum] + dependencies=(Depends(user_api_key_auth),), + response_model=ListResponse[ModelGroupInfoProxy], +) +async def public_model_hub_list( + request: Request, + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +) -> ListResponse[ModelGroupInfoProxy]: + """ + The public model groups this proxy publishes, paged, sortable, searchable and + filterable, for the public Model Hub page. No authentication. + + A rejected request answers with the parameters, sort fields and filter operators + it would have accepted, so the accepted set stays discoverable from the endpoint + itself rather than from a copy of the spec kept here. + + Example curl: + ``` + curl --location --globoff \ + 'http://0.0.0.0:4000/public/v1/model_hub?sort=-input_cost_per_token&filter[mode][in]=chat&page_size=25' + ``` + """ + try: + from litellm.proxy.proxy_server import ( + _get_model_group_info, # pyright: ignore[reportPrivateUsage] # /public/model_hub imports it the same way + llm_router, + prisma_client, + ) + + if llm_router is None: + raise ManagementProblem( + ProblemDetail( + type=f"{PROBLEM_TYPE_BASE}no-llm-router", + title="No models configured", + status=400, + detail=CommonProxyErrors.no_llm_router.value, + ) + ) + + rows: Final[Sequence[ModelGroupInfoProxy]] = ( + () + if litellm.public_model_groups is None + else tuple( + _get_model_group_info( + llm_router=llm_router, + all_models_str=litellm.public_model_groups, + model_group=None, + ) + ) + ) + + return await handle_list( + spec=MODEL_HUB_LIST_SPEC, + executor=_executor(rows, prisma_client), + request=request, + caller=user_api_key_dict, + ) + + except ManagementProblem: + raise + except Exception as e: # noqa: BLE001 # a router error answers as a problem document, not the OpenAI error shape + verbose_proxy_logger.exception( + "litellm.proxy.public_endpoints.public_v1.model_hub.public_model_hub_list(): Exception occured - %s", e + ) + raise ManagementProblem( + ProblemDetail( + type=f"{PROBLEM_TYPE_BASE}internal-server-error", + title="Internal server error", + status=500, + detail="Failed to list public model groups.", + ) + ) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 0f5792e12db..2c571b4027b 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -5833,6 +5833,29 @@ class PrismaClient: verbose_proxy_logger.error("Error getting all latest health checks: %s", e) return [] + async def get_latest_health_checks_for_models( + self, model_names: "Sequence[str]" + ) -> "Sequence[prisma_models.LiteLLM_HealthCheckTable]": + """ + Get the latest health check for each of the named models. + + Same DISTINCT ON as ``get_all_latest_health_checks``, bounded to the models asked + about, so a paged caller reads health for its page instead of for the whole table. + """ + if not model_names: + return () + latest_first: Final = (("model_id", "asc"), ("model_name", "asc"), ("checked_at", "desc")) + order: Final = [{field: direction} for field, direction in latest_first] # mutable-ok: prisma order is a list + try: + return await HealthCheckRepository(self).table.find_many( + where={"model_name": {"in": list(model_names)}}, # mutable-ok: prisma filters are dicts and lists + distinct=["model_id", "model_name"], # mutable-ok: prisma distinct takes a list + order=order, + ) + except Exception as e: # noqa: BLE001 # health decorates a list; a driver error must not fail the page + verbose_proxy_logger.error("Error getting latest health checks for models: %s", e) + return () + ### HELPER FUNCTIONS ### diff --git a/tests/test_litellm/proxy/management_endpoints/management_v1/test_common.py b/tests/test_litellm/proxy/list_api/test_common.py similarity index 85% rename from tests/test_litellm/proxy/management_endpoints/management_v1/test_common.py rename to tests/test_litellm/proxy/list_api/test_common.py index f3515e84d0d..7275b3544fa 100644 --- a/tests/test_litellm/proxy/management_endpoints/management_v1/test_common.py +++ b/tests/test_litellm/proxy/list_api/test_common.py @@ -9,8 +9,8 @@ import pytest from fastapi import Depends, FastAPI, Header, Query, Request from fastapi.testclient import TestClient -import litellm.proxy.management_endpoints.management_v1.common as common_module -from litellm.proxy.management_endpoints.management_v1.common import ( +import litellm.proxy.list_api.common as common_module +from litellm.proxy.list_api.common import ( PROBLEM_CONTENT_TYPE, ManagementProblem, _declared_query_params, @@ -106,7 +106,17 @@ def test_declared_query_params_is_empty_when_the_route_has_no_dependant(): # `fastapi>=0.136.3,<1.0`. Add a name here whenever a supported release drops one. FASTAPI_NAMES_REMOVED_IN_0_140_7 = frozenset({"get_flat_dependant"}) -MANAGEMENT_V1_PACKAGE = Path(str(common_module.__file__)).parent +LIST_API_PACKAGE = Path(str(common_module.__file__)).parent +PROXY_PACKAGE = LIST_API_PACKAGE.parent +GUARDED_PACKAGES = ( + LIST_API_PACKAGE, + PROXY_PACKAGE / "management_endpoints" / "management_v1", + PROXY_PACKAGE / "public_endpoints" / "public_v1", +) +FRAMEWORK_SOURCE_FILES = sorted( + (path for package in GUARDED_PACKAGES for path in package.glob("*.py")), + key=lambda path: (path.parent.name, path.name), +) def _public_names(module: ModuleType) -> frozenset[str]: @@ -123,17 +133,15 @@ def _fastapi_names_imported_by(source_file: Path) -> frozenset[str]: ) -@pytest.mark.parametrize( - "source_file", sorted(MANAGEMENT_V1_PACKAGE.glob("*.py")), ids=lambda path: path.name -) +@pytest.mark.parametrize("source_file", FRAMEWORK_SOURCE_FILES, ids=lambda path: f"{path.parent.name}/{path.name}") def test_no_module_imports_a_fastapi_name_removed_in_a_supported_release(source_file: Path): """`pyproject.toml` allows fastapi up to <1.0, but CI only ever resolves 0.136.3. Every other test here passes just as well against a module importing a name fastapi has since deleted, because the pinned fastapi still has it. On a user's - fastapi>=0.140.7 that import is an ImportError, and `proxy_server` imports this - package unguarded at module level, so it takes the whole proxy down rather than - just these routes. Globbing the package means a new module is covered on sight. + fastapi>=0.140.7 that import is an ImportError, and `proxy_server` imports every one + of these packages unguarded at module level, so it takes the whole proxy down rather + than just these routes. Globbing them means a new module is covered on sight. """ assert not _fastapi_names_imported_by(source_file) & FASTAPI_NAMES_REMOVED_IN_0_140_7 @@ -148,7 +156,7 @@ def test_common_still_imports_when_fastapi_has_dropped_those_names(monkeypatch: for name in FASTAPI_NAMES_REMOVED_IN_0_140_7: monkeypatch.delattr(fastapi_dependency_utils, name, raising=False) spec = importlib.util.spec_from_file_location( - "management_v1_common__simulated_fastapi", Path(str(common_module.__file__)) + "list_api_common__simulated_fastapi", Path(str(common_module.__file__)) ) assert spec is not None and spec.loader is not None reimported = importlib.util.module_from_spec(spec) diff --git a/tests/test_litellm/proxy/list_api/test_in_memory.py b/tests/test_litellm/proxy/list_api/test_in_memory.py new file mode 100644 index 00000000000..efde2949f5c --- /dev/null +++ b/tests/test_litellm/proxy/list_api/test_in_memory.py @@ -0,0 +1,254 @@ +from collections.abc import Sequence +from dataclasses import dataclass +from datetime import datetime, timezone +from types import MappingProxyType + +import pytest + +from litellm.proxy.list_api.in_memory import Cells, InMemoryListExecutor +from litellm.proxy.list_api.list_framework import ( + AnyOf, + Compare, + IsNull, + QueryPlan, + SortKey, + Within, +) + + +@dataclass(frozen=True, slots=True) +class Row: + name: str + size: float | None = None + tags: tuple[str | None, ...] = () + seen_at: datetime | None = None + + +def _cells(row: Row) -> Cells: + return MappingProxyType({"name": row.name, "size": row.size, "tags": row.tags, "seen_at": row.seen_at}) + + +def _executor(*rows: Row, **kwargs) -> InMemoryListExecutor[Row]: + return InMemoryListExecutor(rows=rows, cells=_cells, **kwargs) + + +def _plan(where=(), order=(SortKey(field="name", descending=False),), skip=0, take=50) -> QueryPlan: + return QueryPlan(where=where, order=order, skip=skip, take=take) + + +async def _names(executor: InMemoryListExecutor[Row], plan: QueryPlan) -> list[str]: + return [row.name for row in await executor.find_many(plan)] + + +@pytest.mark.asyncio +async def test_the_page_is_sliced_after_the_sort_not_before(): + executor = _executor(Row("c"), Row("a"), Row("b"), Row("d")) + + assert await _names(executor, _plan(skip=1, take=2)) == ["b", "c"] + + +@pytest.mark.asyncio +async def test_count_ignores_the_page_and_counts_the_match_set(): + executor = _executor(*(Row(f"r{index}") for index in range(7))) + + assert await executor.count(()) == 7 + assert len(await executor.find_many(_plan(take=3))) == 3 + + +@pytest.mark.asyncio +async def test_nulls_sort_last_in_both_directions(): + """`order_by_sql` renders NULLS LAST both ways; an in-memory plan has to agree.""" + executor = _executor(Row("small", size=1.0), Row("unsized"), Row("big", size=9.0)) + + ascending = SortKey(field="size", descending=False) + descending = SortKey(field="size", descending=True) + assert await _names(executor, _plan(order=(ascending,))) == ["small", "big", "unsized"] + assert await _names(executor, _plan(order=(descending,))) == ["big", "small", "unsized"] + + +@pytest.mark.asyncio +async def test_the_last_sort_key_breaks_ties_in_the_first(): + executor = _executor(Row("b", size=1.0), Row("a", size=1.0), Row("c", size=0.0)) + + order = (SortKey(field="size", descending=False), SortKey(field="name", descending=False)) + + assert await _names(executor, _plan(order=order)) == ["c", "a", "b"] + + +@pytest.mark.asyncio +async def test_a_predicate_holds_when_any_element_of_a_repeated_field_matches(): + executor = _executor(Row("azure", tags=("azure", "bedrock")), Row("openai", tags=("openai",))) + + where = (Compare(field="tags", op="contains", value="bedrock"),) + + assert await _names(executor, _plan(where=where)) == ["azure"] + + +@pytest.mark.asyncio +async def test_a_repeated_field_with_no_elements_matches_nothing(): + executor = _executor(Row("untagged")) + + where = (Compare(field="tags", op="contains", value="anything"),) + + assert await _names(executor, _plan(where=where)) == [] + + +@pytest.mark.asyncio +async def test_a_repeated_field_is_matched_element_by_element_not_as_one_string(): + """Without the per-element lift the tuple stringifies, and its punctuation becomes matchable.""" + executor = _executor(Row("azure", tags=("azure", "bedrock"))) + + where = (Compare(field="tags", op="contains", value="e', 'b"),) + + assert await _names(executor, _plan(where=where)) == [] + + +@pytest.mark.asyncio +async def test_within_matches_an_element_of_a_repeated_field(): + executor = _executor(Row("azure", tags=("azure", "bedrock")), Row("openai", tags=("openai",))) + + where = (Within(field="tags", values=("bedrock",)),) + + assert await _names(executor, _plan(where=where)) == ["azure"] + + +@pytest.mark.asyncio +async def test_contains_is_case_insensitive_like_ilike(): + executor = _executor(Row("GPT-5"), Row("claude-opus")) + + where = (Compare(field="name", op="contains", value="gpt"),) + + assert await _names(executor, _plan(where=where)) == ["GPT-5"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("op", ["eq", "not", "gt", "gte", "lt", "lte", "contains"]) +async def test_a_null_cell_satisfies_no_comparison(op: str): + """SQL's three-valued logic: `col <> 1` does not return NULL rows, so neither does this.""" + executor = _executor(Row("unsized")) + + where = (Compare(field="size", op=op, value=1.0),) + + assert await _names(executor, _plan(where=where)) == [] + + +@pytest.mark.asyncio +async def test_is_null_is_the_way_to_ask_for_the_null_rows(): + executor = _executor(Row("unsized"), Row("sized", size=2.0)) + + assert await _names(executor, _plan(where=(IsNull(field="size", negated=False),))) == ["unsized"] + assert await _names(executor, _plan(where=(IsNull(field="size", negated=True),))) == ["sized"] + + +@pytest.mark.asyncio +async def test_is_null_reads_a_repeated_field_element_by_element_too(): + """Every other predicate lifts over a repeated field; `is_null` reading the container + instead would make a field holding only nulls indistinguishable from a populated one.""" + executor = _executor(Row("only_nulls", tags=(None,)), Row("populated", tags=("openai",))) + + assert await _names(executor, _plan(where=(IsNull(field="tags", negated=False),))) == ["only_nulls"] + assert await _names(executor, _plan(where=(IsNull(field="tags", negated=True),))) == ["populated"] + + +@pytest.mark.asyncio +async def test_ordering_comparisons_work_across_the_cell_types(): + when = datetime(2026, 8, 1, tzinfo=timezone.utc) + executor = _executor(Row("early", seen_at=when), Row("late", seen_at=datetime(2026, 9, 1, tzinfo=timezone.utc))) + + where = (Compare(field="seen_at", op="gt", value=when),) + + assert await _names(executor, _plan(where=where)) == ["late"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "op,expected", + [ + ("eq", ["mid"]), + ("not", ["low", "high"]), + ("gt", ["high"]), + ("gte", ["mid", "high"]), + ("lt", ["low"]), + ("lte", ["low", "mid"]), + ], +) +async def test_every_comparison_operator_selects_the_rows_sql_would(op: str, expected: list[str]): + """The endpoint only exposes eq/in/contains today, so without this the ordering + operators are live code no test evaluates.""" + executor = _executor(Row("low", size=1.0), Row("mid", size=2.0), Row("high", size=3.0)) + + where = (Compare(field="size", op=op, value=2.0),) + + assert sorted(await _names(executor, _plan(where=where))) == sorted(expected) + + +@pytest.mark.asyncio +async def test_a_value_of_the_wrong_type_matches_nothing_rather_than_raising(): + executor = _executor(Row("a", size=1.0)) + + where = (Compare(field="size", op="gt", value="not-a-number"),) + + assert await _names(executor, _plan(where=where)) == [] + + +@pytest.mark.asyncio +async def test_within_matches_any_of_its_values(): + executor = _executor(Row("a"), Row("b"), Row("c")) + + where = (Within(field="name", values=("a", "c")),) + + assert await _names(executor, _plan(where=where)) == ["a", "c"] + + +@pytest.mark.asyncio +async def test_any_of_is_a_disjunction_and_the_plan_is_a_conjunction(): + executor = _executor(Row("alpha", size=1.0), Row("beta", size=1.0), Row("alpha-2", size=9.0)) + + where = ( + Compare(field="size", op="lte", value=5.0), + AnyOf(clauses=(Compare(field="name", op="contains", value="alpha"),)), + ) + + assert await _names(executor, _plan(where=where)) == ["alpha"] + + +@pytest.mark.asyncio +async def test_enrich_page_sees_the_page_and_only_the_page(): + seen: list[tuple[str, ...]] = [] + + async def _record(rows: Sequence[Row]) -> Sequence[Row]: + seen.append(tuple(row.name for row in rows)) + return rows + + executor = _executor(*(Row(f"r{index:02d}") for index in range(20)), enrich_page=_record) + + await executor.find_many(_plan(skip=5, take=3)) + + assert seen == [("r05", "r06", "r07")] + + +@pytest.mark.asyncio +async def test_enrich_page_can_replace_the_rows_it_is_given(): + async def _rename(rows: Sequence[Row]) -> Sequence[Row]: + return tuple(Row(f"{row.name}!") for row in rows) + + executor = _executor(Row("a"), Row("b"), enrich_page=_rename) + + assert await _names(executor, _plan()) == ["a!", "b!"] + + +@pytest.mark.asyncio +async def test_counting_never_enriches(): + async def _explode(rows: Sequence[Row]) -> Sequence[Row]: + raise AssertionError("count must not resolve anything a row does not already carry") + + executor = _executor(Row("a"), Row("b"), enrich_page=_explode) + + assert await executor.count(()) == 2 + + +@pytest.mark.asyncio +async def test_rows_pass_through_untouched_without_an_enricher(): + executor = _executor(Row("a"), Row("b")) + + assert await _names(executor, _plan()) == ["a", "b"] diff --git a/tests/test_litellm/proxy/management_endpoints/management_v1/test_list_framework.py b/tests/test_litellm/proxy/list_api/test_list_framework.py similarity index 96% rename from tests/test_litellm/proxy/management_endpoints/management_v1/test_list_framework.py rename to tests/test_litellm/proxy/list_api/test_list_framework.py index 35bd5517361..6ed3ab369c2 100644 --- a/tests/test_litellm/proxy/management_endpoints/management_v1/test_list_framework.py +++ b/tests/test_litellm/proxy/list_api/test_list_framework.py @@ -7,13 +7,12 @@ from fastapi import Request from pydantic import BaseModel from litellm.proxy._types import UserAPIKeyAuth -from litellm.proxy.management_endpoints.management_v1.common import ( - MANAGEMENT_V1_PREFIX, +from litellm.proxy.list_api.common import ( PROBLEM_TYPE_BASE, ManagementProblem, build_page_links, ) -from litellm.proxy.management_endpoints.management_v1.list_framework import ( +from litellm.proxy.list_api.list_framework import ( AnyOf, Compare, FilterSpec, @@ -30,6 +29,7 @@ from litellm.proxy.management_endpoints.management_v1.list_framework import ( order_by_sql, where_sql, ) +from litellm.proxy.management_endpoints.management_v1.common import MANAGEMENT_V1_PREFIX from litellm.types.proxy.management_endpoints.management_v1 import ( PageLinks, PageMeta, @@ -450,6 +450,29 @@ def test_one_bad_key_rejects_the_whole_multi_key_sort(): assert _problem({"sort": "-created_at,api_key"}).type == f"{PROBLEM_TYPE_BASE}invalid-sort-field" +def test_a_repeated_sort_field_is_rejected(): + """An in-memory executor sorts once per key, so a repeat is unbounded work an + unauthenticated caller controls. Rejecting repeats caps it at len(sortable).""" + problem = _problem({"sort": "created_at,max_budget,created_at"}) + + assert problem.status == 400 + assert problem.type == f"{PROBLEM_TYPE_BASE}duplicate-sort-field" + assert "created_at" in problem.detail + assert "max_budget" not in problem.detail + + +def test_a_field_repeated_in_both_directions_is_still_a_repeat(): + assert _problem({"sort": "created_at,-created_at"}).type == f"{PROBLEM_TYPE_BASE}duplicate-sort-field" + + +def test_the_appended_tiebreaker_does_not_count_as_a_repeat(): + """The tiebreaker is added after parsing, so sorting by it explicitly stays legal.""" + assert _plan({"sort": "-budget_id"}).order == ( + SortKey(field="budget_id", descending=True), + SortKey(field="budget_id", descending=False), + ) + + def test_a_double_dash_prefix_is_not_a_descending_sort(): assert _problem({"sort": "--created_at"}).type == f"{PROBLEM_TYPE_BASE}invalid-sort-field" diff --git a/tests/test_litellm/proxy/management_endpoints/management_v1/test_budgets.py b/tests/test_litellm/proxy/management_endpoints/management_v1/test_budgets.py index 40473f1a25a..add2126ac7b 100644 --- a/tests/test_litellm/proxy/management_endpoints/management_v1/test_budgets.py +++ b/tests/test_litellm/proxy/management_endpoints/management_v1/test_budgets.py @@ -10,22 +10,22 @@ from fastapi.testclient import TestClient from litellm.proxy._types import LiteLLMRoutes, LitellmUserRoles from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth, user_api_key_auth +from litellm.proxy.list_api.common import ( + PROBLEM_TYPE_BASE, + ManagementProblem, + problem_response, +) +from litellm.proxy.list_api.list_framework import ( + Compare, + ScopeWhere, + build_query_plan, +) from litellm.proxy.management_endpoints.management_v1 import router from litellm.proxy.management_endpoints.management_v1.budgets import ( BUDGETS_LIST_SPEC, BudgetListItem, ) -from litellm.proxy.management_endpoints.management_v1.common import ( - MANAGEMENT_V1_PREFIX, - PROBLEM_TYPE_BASE, - ManagementProblem, - problem_response, -) -from litellm.proxy.management_endpoints.management_v1.list_framework import ( - Compare, - ScopeWhere, - build_query_plan, -) +from litellm.proxy.management_endpoints.management_v1.common import MANAGEMENT_V1_PREFIX from litellm.types.proxy.management_endpoints.management_v1 import ProblemDetail app = FastAPI() diff --git a/tests/test_litellm/proxy/management_endpoints/management_v1/test_spend_logs.py b/tests/test_litellm/proxy/management_endpoints/management_v1/test_spend_logs.py index 35fcd3b6cd7..b6867d338c5 100644 --- a/tests/test_litellm/proxy/management_endpoints/management_v1/test_spend_logs.py +++ b/tests/test_litellm/proxy/management_endpoints/management_v1/test_spend_logs.py @@ -8,13 +8,13 @@ from fastapi.testclient import TestClient from litellm.proxy._types import LiteLLMRoutes, LitellmUserRoles from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth, user_api_key_auth -from litellm.proxy.management_endpoints.management_v1 import router -from litellm.proxy.management_endpoints.management_v1.common import ( - MANAGEMENT_V1_PREFIX, +from litellm.proxy.list_api.common import ( PROBLEM_TYPE_BASE, ManagementProblem, problem_response, ) +from litellm.proxy.management_endpoints.management_v1 import router +from litellm.proxy.management_endpoints.management_v1.common import MANAGEMENT_V1_PREFIX from litellm.types.proxy.management_endpoints.management_v1 import ProblemDetail app = FastAPI() diff --git a/tests/test_litellm/proxy/public_endpoints/public_v1/test_model_hub.py b/tests/test_litellm/proxy/public_endpoints/public_v1/test_model_hub.py new file mode 100644 index 00000000000..631e91dca11 --- /dev/null +++ b/tests/test_litellm/proxy/public_endpoints/public_v1/test_model_hub.py @@ -0,0 +1,349 @@ +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from datetime import datetime, timezone +from types import MappingProxyType +from unittest.mock import AsyncMock, MagicMock + +import pytest +from fastapi.testclient import TestClient + +import litellm +from litellm.proxy._types import LiteLLMRoutes +from litellm.proxy.proxy_server import app +from litellm.types.router import ModelGroupInfo + +client = TestClient(app) + +MODEL_HUB_PATH = "/public/v1/model_hub" +LEGACY_MODEL_HUB_PATH = "/public/model_hub" + + +@dataclass(frozen=True, slots=True) +class _FakeRouter: + """Stands in for the running Router: `_get_model_group_info` only ever asks it this.""" + + infos: Mapping[str, ModelGroupInfo] + + def get_model_group_info(self, model_group: str) -> ModelGroupInfo | None: + return self.infos.get(model_group) + + +def _info( + name: str, + *, + mode: str = "chat", + providers: Sequence[str] = ("openai",), + **overrides: object, +) -> ModelGroupInfo: + return ModelGroupInfo(model_group=name, mode=mode, providers=list(providers), **overrides) + + +def _publish(monkeypatch, infos: Sequence[ModelGroupInfo], prisma_client: object | None = None) -> None: + monkeypatch.setattr(litellm, "public_model_groups", [info.model_group for info in infos]) + monkeypatch.setattr( + "litellm.proxy.proxy_server.llm_router", + _FakeRouter(infos=MappingProxyType({info.model_group: info for info in infos})), + ) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", prisma_client) + + +def _named(count: int, **overrides: object) -> Sequence[ModelGroupInfo]: + return tuple(_info(f"model-{index:03d}", **overrides) for index in range(count)) + + +def _get(query: str = "", **kwargs): + suffix = f"?{query}" if query else "" + return client.get(f"{MODEL_HUB_PATH}{suffix}", **kwargs) + + +def _groups(response) -> list[str]: + return [row["model_group"] for row in response.json()["data"]] + + +def _health_check(model_name: str, status: str = "healthy"): + check = MagicMock() + check.model_name = model_name + check.model_id = None + check.status = status + check.response_time_ms = 12.5 + check.checked_at = datetime(2026, 8, 1, 9, 30, tzinfo=timezone.utc) + return check + + +def _recording_prisma(checks: Sequence[object] = ()): + """A prisma client whose only exercised call is the health-check read, recorded for assertions.""" + read = AsyncMock(return_value=list(checks)) + prisma_client = MagicMock() + prisma_client.get_latest_health_checks_for_models = read + return prisma_client, read + + +def _asked_about(read) -> list[str]: + return list(read.call_args.args[0]) if read.call_args.args else list(read.call_args.kwargs["model_names"]) + + +def test_the_route_is_registered_as_a_public_route(): + """`public_routes` membership is an exact-string check, so the path has to match literally.""" + assert MODEL_HUB_PATH in LiteLLMRoutes.public_routes.value + + +def test_a_page_slices_the_published_model_groups(monkeypatch): + _publish(monkeypatch, _named(120)) + + response = _get("page=2&page_size=25") + + assert response.status_code == 200, response.text + assert _groups(response) == [f"model-{index:03d}" for index in range(25, 50)] + assert response.json()["meta"] == {"total_count": 120, "page": 2, "page_size": 25, "total_pages": 5} + + +def test_every_page_link_resolves_to_the_page_it_names(monkeypatch): + _publish(monkeypatch, _named(120)) + + links = _get("page=2&page_size=25").json()["links"] + + assert client.get(links["first"]).json()["meta"]["page"] == 1 + assert client.get(links["prev"]).json()["meta"]["page"] == 1 + assert client.get(links["self"]).json()["meta"]["page"] == 2 + assert client.get(links["next"]).json()["meta"]["page"] == 3 + assert client.get(links["last"]).json()["meta"]["page"] == 5 + + +def test_total_count_counts_the_whole_match_set_not_the_page(monkeypatch): + _publish(monkeypatch, (*_named(30), _info("embedder-1", mode="embedding"))) + + response = _get("filter[mode]=chat&page_size=5") + + assert len(response.json()["data"]) == 5 + assert response.json()["meta"]["total_count"] == 30 + + +def test_health_is_resolved_only_for_the_rows_on_the_page(monkeypatch): + """The bug this endpoint exists to fix: enriching before slicing costs the whole collection. + + An enrich-then-slice implementation asks about all 200 model groups here, not the 10 served. + """ + prisma_client, read = _recording_prisma() + _publish(monkeypatch, _named(200), prisma_client=prisma_client) + + response = _get("page=1&page_size=10") + + assert len(response.json()["data"]) == 10 + assert _asked_about(read) == [f"model-{index:03d}" for index in range(10)] + + +def test_health_is_asked_about_the_second_page_not_the_first(monkeypatch): + prisma_client, read = _recording_prisma() + _publish(monkeypatch, _named(200), prisma_client=prisma_client) + + _get("page=4&page_size=10") + + assert _asked_about(read) == [f"model-{index:03d}" for index in range(30, 40)] + + +def test_the_latest_health_check_lands_on_its_row(monkeypatch): + prisma_client, _ = _recording_prisma([_health_check("model-001", status="unhealthy")]) + _publish(monkeypatch, _named(3), prisma_client=prisma_client) + + rows = {row["model_group"]: row for row in _get().json()["data"]} + + assert rows["model-001"]["health_status"] == "unhealthy" + assert rows["model-001"]["health_response_time"] == 12.5 + assert rows["model-001"]["health_checked_at"] == "2026-08-01T09:30:00+00:00" + assert rows["model-000"]["health_status"] is None + + +def test_a_health_read_that_returns_nothing_still_serves_the_page(monkeypatch): + prisma_client, read = _recording_prisma() + read.return_value = [] + _publish(monkeypatch, _named(3), prisma_client=prisma_client) + + response = _get() + + assert response.status_code == 200, response.text + assert _groups(response) == ["model-000", "model-001", "model-002"] + + +def test_rows_are_alphabetical_by_default(monkeypatch): + _publish(monkeypatch, (_info("zeta"), _info("alpha"), _info("mid"))) + + assert _groups(_get()) == ["alpha", "mid", "zeta"] + + +def test_a_descending_sort_reverses_the_order(monkeypatch): + _publish(monkeypatch, (_info("zeta"), _info("alpha"), _info("mid"))) + + assert _groups(_get("sort=-model_group")) == ["zeta", "mid", "alpha"] + + +def test_sorting_by_a_numeric_field_puts_the_unset_ones_last_in_both_directions(monkeypatch): + _publish( + monkeypatch, + ( + _info("cheap", input_cost_per_token=0.000001), + _info("unpriced"), + _info("dear", input_cost_per_token=0.00003), + ), + ) + + assert _groups(_get("sort=input_cost_per_token")) == ["cheap", "dear", "unpriced"] + assert _groups(_get("sort=-input_cost_per_token")) == ["dear", "cheap", "unpriced"] + + +def test_an_undeclared_sort_field_is_a_problem_naming_the_allowed_fields(monkeypatch): + _publish(monkeypatch, _named(3)) + + response = _get("sort=providers") + + assert response.status_code == 400 + assert response.headers["content-type"].startswith("application/problem+json") + body = response.json() + assert "providers" in body["detail"] + assert body["allowed"] == [ + "input_cost_per_token", + "max_input_tokens", + "max_output_tokens", + "mode", + "model_group", + "output_cost_per_token", + ] + + +def test_a_repeated_sort_field_is_rejected_rather_than_sorted_twice(monkeypatch): + """The route is unauthenticated and sorts in memory once per key, so an unbounded + key list is CPU any caller can spend.""" + _publish(monkeypatch, _named(3)) + + response = _get("sort=model_group,model_group") + + assert response.status_code == 400 + assert response.headers["content-type"].startswith("application/problem+json") + assert response.json()["type"] == "urn:litellm:error:duplicate-sort-field" + + +def test_an_unknown_query_parameter_is_a_problem_outside_management_v1(monkeypatch): + """The `ManagementProblem` handler is registered on the app, not on the `/management/v1` prefix.""" + _publish(monkeypatch, _named(3)) + + response = _get("limit=10") + + assert response.status_code == 400 + assert response.headers["content-type"].startswith("application/problem+json") + assert "limit" in response.json()["detail"] + + +def test_a_repeated_query_parameter_is_rejected(monkeypatch): + _publish(monkeypatch, _named(3)) + + response = _get("page=1&page=99") + + assert response.status_code == 400 + assert "page" in response.json()["detail"] + + +def test_a_mode_filter_narrows_the_list(monkeypatch): + _publish(monkeypatch, (_info("chatter"), _info("embedder", mode="embedding"))) + + assert _groups(_get("filter[mode]=embedding")) == ["embedder"] + assert _groups(_get("filter[mode][in]=chat,embedding")) == ["chatter", "embedder"] + + +def test_a_provider_filter_matches_a_model_group_serving_that_provider(monkeypatch): + _publish( + monkeypatch, + ( + _info("openai-only"), + _info("mixed", providers=["azure", "bedrock"]), + ), + ) + + assert _groups(_get("filter[providers][contains]=bedrock")) == ["mixed"] + assert _groups(_get("filter[providers][contains]=openai")) == ["openai-only"] + assert _groups(_get("filter[providers][contains]=e, b")) == [] + + +def test_the_search_matches_model_group_names_case_insensitively(monkeypatch): + _publish(monkeypatch, (_info("gpt-4o"), _info("claude-opus"), _info("GPT-5"))) + + assert _groups(_get("q=gpt")) == ["GPT-5", "gpt-4o"] + + +@pytest.fixture +def guarded(monkeypatch): + """A proxy with a master key set, so anything but a public route would demand credentials.""" + monkeypatch.setattr("litellm.proxy.proxy_server.master_key", "sk-1234") + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {}) + + +def test_an_unauthenticated_caller_is_served(monkeypatch, guarded): + _publish(monkeypatch, _named(2)) + + response = _get() + + assert response.status_code == 200, response.text + assert len(response.json()["data"]) == 2 + + +def test_a_bad_api_key_does_not_turn_a_public_route_into_a_401(monkeypatch, guarded): + _publish(monkeypatch, _named(2)) + + response = _get(headers={"Authorization": "Bearer sk-definitely-not-a-real-key"}) + + assert response.status_code == 200, response.text + assert len(response.json()["data"]) == 2 + + +def test_no_published_model_groups_yields_an_empty_but_coherent_envelope(monkeypatch): + _publish(monkeypatch, ()) + monkeypatch.setattr(litellm, "public_model_groups", None) + + response = _get() + + assert response.status_code == 200, response.text + body = response.json() + assert body["data"] == [] + assert body["meta"] == {"total_count": 0, "page": 1, "page_size": 50, "total_pages": 0} + assert body["links"]["first"].endswith("page=1") + assert body["links"]["last"].endswith("page=1") + assert body["links"]["next"] is None + assert body["links"]["prev"] is None + + +def test_no_router_answers_with_a_problem_rather_than_the_openai_error_shape(monkeypatch): + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None) + + response = _get() + + assert response.status_code == 400 + assert response.headers["content-type"].startswith("application/problem+json") + assert response.json()["type"] == "urn:litellm:error:no-llm-router" + + +def test_an_unexpected_router_failure_answers_as_a_problem_not_the_openai_error_shape(monkeypatch): + class _Exploding: + def get_model_group_info(self, model_group: str) -> ModelGroupInfo: + raise RuntimeError("router blew up") + + monkeypatch.setattr(litellm, "public_model_groups", ["boom"]) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", _Exploding()) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + + response = _get() + + assert response.status_code == 500 + assert response.headers["content-type"].startswith("application/problem+json") + assert response.json()["type"] == "urn:litellm:error:internal-server-error" + + +@pytest.mark.parametrize("query", ["", "page=1&page_size=2"]) +def test_the_endpoint_it_supersedes_still_answers_with_its_bare_array(monkeypatch, query: str): + """`/public/model_hub` is what the shipped UI calls; this PR must not move it at all.""" + _publish(monkeypatch, _named(3)) + suffix = f"?{query}" if query else "" + + response = client.get(f"{LEGACY_MODEL_HUB_PATH}{suffix}") + + assert response.status_code == 200, response.text + body = response.json() + assert isinstance(body, list) + assert [row["model_group"] for row in body] == ["model-000", "model-001", "model-002"] diff --git a/tests/test_litellm/proxy/utils/prisma_and_spend/test_prisma_client_health.py b/tests/test_litellm/proxy/utils/prisma_and_spend/test_prisma_client_health.py index 220fff1a881..9f48ba68b4f 100644 --- a/tests/test_litellm/proxy/utils/prisma_and_spend/test_prisma_client_health.py +++ b/tests/test_litellm/proxy/utils/prisma_and_spend/test_prisma_client_health.py @@ -9,6 +9,7 @@ Symbols pinned here: - ``PrismaClient.save_health_check_result`` - ``PrismaClient.get_health_check_history`` - ``PrismaClient.get_all_latest_health_checks`` + - ``PrismaClient.get_latest_health_checks_for_models`` - ``PrismaClient._is_sha256_hex`` (a nested helper inside ``migrate_passwords_to_scrypt_async``; the pin list assigns it to this cluster as a documentation artifact) @@ -290,3 +291,40 @@ async def test_get_all_latest_health_checks_db_error_returns_empty_list( side_effect=RuntimeError("oops") ) assert await prisma_client.get_all_latest_health_checks() == [] + + +@pytest.mark.asyncio +async def test_get_latest_health_checks_for_models_bounds_the_query_to_those_models( + prisma_client: PrismaClient, +) -> None: + """A paged caller reads health for its page; an unbounded read is the bug this exists to avoid.""" + prisma_client.db.litellm_healthchecktable.find_many = AsyncMock(return_value=[]) + await prisma_client.get_latest_health_checks_for_models(["gpt-5", "claude-opus"]) + kwargs = prisma_client.db.litellm_healthchecktable.find_many.await_args.kwargs + actual = { + "where": kwargs["where"], + "distinct": kwargs["distinct"], + "order": kwargs["order"], + } + assert actual == { + "where": {"model_name": {"in": ["gpt-5", "claude-opus"]}}, + "distinct": ["model_id", "model_name"], + "order": [{"model_id": "asc"}, {"model_name": "asc"}, {"checked_at": "desc"}], + } + + +@pytest.mark.asyncio +async def test_get_latest_health_checks_for_models_does_not_query_for_an_empty_page( + prisma_client: PrismaClient, +) -> None: + prisma_client.db.litellm_healthchecktable.find_many = AsyncMock(return_value=[]) + assert await prisma_client.get_latest_health_checks_for_models([]) == () + assert prisma_client.db.litellm_healthchecktable.find_many.await_count == 0 + + +@pytest.mark.asyncio +async def test_get_latest_health_checks_for_models_db_error_returns_empty_list( + prisma_client: PrismaClient, +) -> None: + prisma_client.db.litellm_healthchecktable.find_many = AsyncMock(side_effect=RuntimeError("oops")) + assert await prisma_client.get_latest_health_checks_for_models(["gpt-5"]) == () diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 1e1ce665970..55f79633da3 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -12061,6 +12061,36 @@ export interface paths { patch?: never; trace?: never; }; + "/public/v1/model_hub": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Public Model Hub List + * @description The public model groups this proxy publishes, paged, sortable, searchable and + * filterable, for the public Model Hub page. No authentication. + * + * A rejected request answers with the parameters, sort fields and filter operators + * it would have accepted, so the accepted set stays discoverable from the endpoint + * itself rather than from a copy of the spec kept here. + * + * Example curl: + * ``` + * curl --location --globoff 'http://0.0.0.0:4000/public/v1/model_hub?sort=-input_cost_per_token&filter[mode][in]=chat&page_size=25' + * ``` + */ + get: operations["public_model_hub_list_public_v1_model_hub_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/queue/chat/completions": { parameters: { query?: never; @@ -27457,6 +27487,13 @@ export interface components { links: components["schemas"]["ListLinks"]; meta: components["schemas"]["ListMeta"]; }; + /** ListResponse[ModelGroupInfoProxy] */ + ListResponse_ModelGroupInfoProxy_: { + /** Data */ + data: components["schemas"]["ModelGroupInfoProxy"][]; + links: components["schemas"]["ListLinks"]; + meta: components["schemas"]["ListMeta"]; + }; /** * ListRunsResponse * @description Response from listing runs @@ -53373,6 +53410,26 @@ export interface operations { }; }; }; + public_model_hub_list_public_v1_model_hub_get: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ListResponse_ModelGroupInfoProxy_"]; + }; + }; + }; + }; async_queue_request_queue_chat_completions_post: { parameters: { query?: {