diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py
index 804ad744989..d703c378ce8 100644
--- a/litellm/proxy/_types.py
+++ b/litellm/proxy/_types.py
@@ -2348,6 +2348,14 @@ class PassThroughGenericEndpoint(LiteLLMPydanticObjectBase):
default=None,
description="List of HTTP methods this endpoint handles (e.g., ['GET', 'POST']). If None or empty, all methods (GET, POST, PUT, DELETE, PATCH) are supported for backward compatibility. This allows the same path to have different targets for different HTTP methods.",
)
+ display_name: str | None = Field(
+ default=None,
+ description="Human-readable name for this endpoint. The Model Hub lists the endpoint under this name, falling back to the path when unset.",
+ )
+ show_in_model_hub: bool = Field(
+ default=False,
+ description="If True, this endpoint is published on the public Model Hub as a row in mode 'passthrough'.",
+ )
class PassThroughEndpointResponse(LiteLLMPydanticObjectBase):
diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py
index 8a728a7335a..b2caa4d837d 100644
--- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py
+++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py
@@ -4748,7 +4748,7 @@ async def update_pass_through_endpoints(
# Find the index for updating the list
endpoint_index = None
for idx, endpoint in enumerate(pass_through_endpoint_data):
- _endpoint = PassThroughGenericEndpoint(**endpoint) if isinstance(endpoint, dict) else endpoint
+ _endpoint = PassThroughGenericEndpoint.model_validate(endpoint) if isinstance(endpoint, dict) else endpoint
if _endpoint.id == endpoint_id:
endpoint_index = idx
break
@@ -4981,7 +4981,7 @@ async def delete_pass_through_endpoints(
# Find the index for deleting from the list
endpoint_index = None
for idx, endpoint in enumerate(pass_through_endpoint_data):
- _endpoint = PassThroughGenericEndpoint(**endpoint) if isinstance(endpoint, dict) else endpoint
+ _endpoint = PassThroughGenericEndpoint.model_validate(endpoint) if isinstance(endpoint, dict) else endpoint
if _endpoint.id == endpoint_id:
endpoint_index = idx
break
@@ -5029,7 +5029,7 @@ def _find_endpoint_by_id(
for endpoint in endpoints_data:
_endpoint: PassThroughGenericEndpoint | None = None
if isinstance(endpoint, dict):
- _endpoint = PassThroughGenericEndpoint(**endpoint)
+ _endpoint = PassThroughGenericEndpoint.model_validate(endpoint)
elif isinstance(endpoint, PassThroughGenericEndpoint):
_endpoint = endpoint
diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py
index 7fd114f9838..e33966ceb29 100644
--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@@ -5858,6 +5858,16 @@ def _reload_settings_store(section: Section, store: SettingsStore, section_confi
store["pass_through_endpoints"] = serving_pass_throughs
+def _clear_db_settings_beside_pass_through_endpoints(section: Section, store: SettingsStore) -> None:
+ stored_pass_throughs: Final = store.db_value("pass_through_endpoints")
+ store.apply_db_row(
+ section,
+ MappingProxyType({"pass_through_endpoints": stored_pass_throughs})
+ if isinstance(stored_pass_throughs, list)
+ else _EMPTY_SETTINGS_MAPPING,
+ )
+
+
def _general_settings_row_is_readable(row: _ConfigParamValueRow | None) -> bool:
return row is None or row.param_value is None or isinstance(row.param_value, Mapping)
@@ -6708,7 +6718,7 @@ class ProxyConfig:
)
else:
for section, store in self._settings_stores.items():
- store.apply_db_row(section, _EMPTY_SETTINGS_MAPPING)
+ _clear_db_settings_beside_pass_through_endpoints(section, store)
config = self._config_with_resolved_settings(config)
diff --git a/litellm/proxy/public_endpoints/model_hub_rows.py b/litellm/proxy/public_endpoints/model_hub_rows.py
new file mode 100644
index 00000000000..9d1d48ad1b8
--- /dev/null
+++ b/litellm/proxy/public_endpoints/model_hub_rows.py
@@ -0,0 +1,86 @@
+"""The rows the public Model Hub lists: published model groups and opted-in pass-through endpoints."""
+
+from collections.abc import Iterator
+from dataclasses import dataclass
+from typing import Final, TypeAlias
+
+from pydantic import TypeAdapter, ValidationError
+
+import litellm
+from litellm.proxy._types import PassThroughGenericEndpoint
+from litellm.types.proxy.management_endpoints.model_management_endpoints import ModelGroupInfoProxy
+
+PASS_THROUGH_MODE: Final = "passthrough"
+_SERVING_ENTRY: Final = TypeAdapter(dict[str, object])
+
+
+@dataclass(frozen=True, slots=True)
+class ModelGroupRow:
+ info: ModelGroupInfoProxy
+
+
+@dataclass(frozen=True, slots=True)
+class PassThroughRow:
+ info: ModelGroupInfoProxy
+
+
+HubRow: TypeAlias = ModelGroupRow | PassThroughRow
+
+
+@dataclass(frozen=True, slots=True)
+class NoModelsConfigured:
+ """Neither a router nor a published pass-through, so the hub has nothing to list."""
+
+
+def pass_through_row(endpoint: PassThroughGenericEndpoint) -> PassThroughRow:
+ return PassThroughRow(
+ info=ModelGroupInfoProxy(
+ model_group=endpoint.display_name or endpoint.path,
+ providers=[],
+ mode=PASS_THROUGH_MODE,
+ is_public_model_group=True,
+ pass_through_path=endpoint.path,
+ pass_through_methods=tuple(endpoint.methods) if endpoint.methods is not None else None,
+ )
+ )
+
+
+def _serving_pass_through_endpoints() -> Iterator[PassThroughGenericEndpoint]:
+ """The endpoints this worker serves, from the in-memory general settings the config sync keeps current."""
+ from litellm.proxy.proxy_server import proxy_config
+
+ configured: Final = proxy_config.settings.get("pass_through_endpoints")
+ for entry in configured if isinstance(configured, list) else ():
+ try:
+ yield PassThroughGenericEndpoint.model_validate(_SERVING_ENTRY.validate_python(entry))
+ except ValidationError:
+ continue
+
+
+def _published_model_group_rows() -> tuple[ModelGroupRow, ...] | None:
+ """None when there is no router, which differs from a router that publishes nothing."""
+ from litellm.proxy.proxy_server import (
+ get_model_group_info,
+ llm_router,
+ )
+
+ if llm_router is None:
+ return None
+ if litellm.public_model_groups is None:
+ return ()
+ return tuple(
+ ModelGroupRow(info=info)
+ for info in get_model_group_info(
+ llm_router=llm_router, all_models_str=litellm.public_model_groups, model_group=None
+ )
+ )
+
+
+def published_hub_rows() -> tuple[HubRow, ...] | NoModelsConfigured:
+ pass_through_rows: Final = tuple(
+ pass_through_row(endpoint) for endpoint in _serving_pass_through_endpoints() if endpoint.show_in_model_hub
+ )
+ model_group_rows: Final = _published_model_group_rows()
+ if model_group_rows is None and not pass_through_rows:
+ return NoModelsConfigured()
+ return (*(model_group_rows or ()), *pass_through_rows)
diff --git a/litellm/proxy/public_endpoints/public_endpoints.py b/litellm/proxy/public_endpoints/public_endpoints.py
index cb1e51d0deb..45abd3c4d5f 100644
--- a/litellm/proxy/public_endpoints/public_endpoints.py
+++ b/litellm/proxy/public_endpoints/public_endpoints.py
@@ -5,6 +5,7 @@ import re
from collections.abc import Awaitable, Callable, Mapping, Sequence
from functools import cache
from importlib.resources import files
+from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Protocol
from fastapi import APIRouter, HTTPException, Request
@@ -25,6 +26,13 @@ from litellm.proxy._types import (
)
from litellm.proxy.db.health_check_latest import latest_by_model_name
from litellm.proxy.deps.repositories import PluginUploadRepositoryDep
+from litellm.proxy.public_endpoints.model_hub_rows import (
+ HubRow,
+ ModelGroupRow,
+ NoModelsConfigured,
+ PassThroughRow,
+ published_hub_rows,
+)
from litellm.proxy.utils import get_custom_url
from litellm.repositories.plugin_publication import read_plugin_publisher, read_plugin_review
from litellm.repositories.table_repositories import ClaudeCodePluginRepository
@@ -276,45 +284,52 @@ def _agent_create_fields() -> tuple[AgentCreateInfo, ...]:
response_model=list[ModelGroupInfoProxy],
)
async def public_model_hub():
- import litellm
- from litellm.proxy.health_endpoints._health_endpoints import (
- convert_health_check_to_dict,
- )
- from litellm.proxy.proxy_server import (
- get_model_group_info,
- llm_router,
- prisma_client,
- )
-
- if llm_router is None:
+ rows: Final = published_hub_rows()
+ if isinstance(rows, NoModelsConfigured):
raise HTTPException(status_code=400, detail=CommonProxyErrors.no_llm_router.value)
- model_groups: list[ModelGroupInfoProxy] = []
- if litellm.public_model_groups is not None:
- model_groups = get_model_group_info( # rebind-ok: pre-existing rebinding on a rename-only line
- llm_router=llm_router,
- all_models_str=litellm.public_model_groups,
- model_group=None,
+ health_checks_map: Final = await _legacy_health_checks()
+ return [_with_legacy_health(row, health_checks_map) for row in rows]
+
+
+async def _legacy_health_checks() -> Mapping[str, Mapping[str, object]]:
+ from litellm.proxy.proxy_server import prisma_client
+
+ if prisma_client is None:
+ return MappingProxyType({})
+ try:
+ latest_checks: Final = await prisma_client.get_all_latest_health_checks()
+ return MappingProxyType(
+ {
+ model_name: MappingProxyType(
+ {
+ "status": check.status,
+ "response_time_ms": check.response_time_ms,
+ "checked_at": check.checked_at.isoformat() if check.checked_at else None,
+ }
+ )
+ for model_name, check in latest_by_model_name(latest_checks).items()
+ }
)
+ except Exception:
+ return MappingProxyType({})
- # Fetch health check information if available
- health_checks_map: Final = {}
- if prisma_client is not None:
- try:
- latest_checks: Final = await prisma_client.get_all_latest_health_checks()
- for model_name, check in latest_by_model_name(latest_checks).items():
- health_checks_map[model_name] = convert_health_check_to_dict(check)
- except Exception:
- pass
- for model_group in model_groups:
- health_info = health_checks_map.get(model_group.model_group)
- if health_info:
- model_group.health_status = health_info.get("status")
- model_group.health_response_time = health_info.get("response_time_ms")
- model_group.health_checked_at = health_info.get("checked_at")
-
- return model_groups
+def _with_legacy_health(row: HubRow, health_checks_map: Mapping[str, Mapping[str, object]]) -> ModelGroupInfoProxy:
+ match row:
+ case PassThroughRow():
+ return row.info
+ case ModelGroupRow():
+ health_info: Final = health_checks_map.get(row.info.model_group)
+ if not health_info:
+ return row.info
+ return row.info.model_copy(
+ update={
+ "health_status": health_info.get("status"),
+ "health_response_time": health_info.get("response_time_ms"),
+ "health_checked_at": health_info.get("checked_at"),
+ }
+ )
@router.get(
diff --git a/litellm/proxy/public_endpoints/public_v1/model_hub.py b/litellm/proxy/public_endpoints/public_v1/model_hub.py
index e58149fba0b..a00138c94c9 100644
--- a/litellm/proxy/public_endpoints/public_v1/model_hub.py
+++ b/litellm/proxy/public_endpoints/public_v1/model_hub.py
@@ -8,7 +8,6 @@ from typing import Annotated, Final, Literal, Protocol
from fastapi import APIRouter, Depends, Request
from typing_extensions import ReadOnly, TypedDict
-import litellm
from litellm._logging import verbose_proxy_logger
from litellm.proxy._types import CommonProxyErrors, UserAPIKeyAuth
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
@@ -24,6 +23,13 @@ from litellm.proxy.list_api.list_framework import (
handle_facet,
handle_list,
)
+from litellm.proxy.public_endpoints.model_hub_rows import (
+ HubRow,
+ ModelGroupRow,
+ NoModelsConfigured,
+ PassThroughRow,
+ published_hub_rows,
+)
from litellm.proxy.utils import PrismaClient
from litellm.types.proxy.management_endpoints.management_v1 import (
FacetListResponse,
@@ -93,9 +99,19 @@ class HealthEnricher:
lookup: HealthSnapshotLookup
- async def __call__(self, rows: Sequence[ModelGroupInfoProxy]) -> Sequence[ModelGroupInfoProxy]:
- health: Final = await self.lookup.latest_for(tuple(row.model_group for row in rows))
- return tuple(_with_health(row, health.get(row.model_group)) for row in rows)
+ async def __call__(self, rows: Sequence[HubRow]) -> Sequence[HubRow]:
+ health: Final = await self.lookup.latest_for(
+ tuple(row.info.model_group for row in rows if isinstance(row, ModelGroupRow))
+ )
+ return tuple(_row_with_health(row, health) for row in rows)
+
+
+def _row_with_health(row: HubRow, health: Mapping[str, HealthSnapshot]) -> HubRow:
+ match row:
+ case PassThroughRow():
+ return row
+ case ModelGroupRow():
+ return ModelGroupRow(info=_with_health(row.info, health.get(row.info.model_group)))
FEATURE_PREFIX: Final = "supports_"
@@ -120,26 +136,26 @@ def _enabled_features(fields: Mapping[str, object]) -> tuple[str, ...]:
)
-def _cells(row: ModelGroupInfoProxy) -> Cells:
+def _cells(row: HubRow) -> Cells:
+ info: Final = row.info
return MappingProxyType(
{
- "model_group": row.model_group,
- "mode": row.mode,
- "providers": tuple(row.providers),
- "features": _features(row),
- "max_input_tokens": row.max_input_tokens,
- "max_output_tokens": row.max_output_tokens,
- "input_cost_per_token": row.input_cost_per_token,
- "output_cost_per_token": row.output_cost_per_token,
- "rpm": row.rpm,
- "tpm": row.tpm,
+ "model_group": info.model_group,
+ "mode": info.mode,
+ "providers": tuple(info.providers),
+ "features": _features(info),
+ "max_input_tokens": info.max_input_tokens,
+ "max_output_tokens": info.max_output_tokens,
+ "input_cost_per_token": info.input_cost_per_token,
+ "output_cost_per_token": info.output_cost_per_token,
+ "rpm": info.rpm,
+ "tpm": info.tpm,
}
)
-def _serialize(row: ModelGroupInfoProxy) -> ModelGroupInfoProxy:
- """The row shape is the wire shape: the rows served are the router's own model group records."""
- return row
+def _serialize(row: HubRow) -> ModelGroupInfoProxy:
+ return row.info
def _scope(_caller: UserAPIKeyAuth) -> Scope:
@@ -163,7 +179,7 @@ MODEL_HUB_FACETS: Final[Mapping[str, str]] = MappingProxyType(
{"providers": "providers", "modes": "mode", "features": "features"}
)
-MODEL_HUB_LIST_SPEC: Final[ListSpec[ModelGroupInfoProxy, ModelGroupInfoProxy]] = ListSpec(
+MODEL_HUB_LIST_SPEC: Final[ListSpec[HubRow, ModelGroupInfoProxy]] = ListSpec(
resource="model groups",
sortable=frozenset(
(
@@ -189,13 +205,9 @@ MODEL_HUB_LIST_SPEC: Final[ListSpec[ModelGroupInfoProxy, ModelGroupInfoProxy]] =
)
-def _published_rows() -> Sequence[ModelGroupInfoProxy]:
- from litellm.proxy.proxy_server import (
- get_model_group_info, # pyright: ignore[reportPrivateUsage] # /public/model_hub imports it the same way
- llm_router,
- )
-
- if llm_router is None:
+def _published_rows() -> Sequence[HubRow]:
+ rows: Final = published_hub_rows()
+ if isinstance(rows, NoModelsConfigured):
raise ManagementProblem(
ProblemDetail(
type=f"{PROBLEM_TYPE_BASE}no-llm-router",
@@ -204,21 +216,13 @@ def _published_rows() -> Sequence[ModelGroupInfoProxy]:
detail=CommonProxyErrors.no_llm_router.value,
)
)
- if litellm.public_model_groups is None:
- return ()
- return tuple(
- get_model_group_info(
- llm_router=llm_router,
- all_models_str=litellm.public_model_groups,
- model_group=None,
- )
- )
+ return rows
def _executor(
- rows: Sequence[ModelGroupInfoProxy],
+ rows: Sequence[HubRow],
prisma_client: PrismaClient | None,
-) -> InMemoryListExecutor[ModelGroupInfoProxy]:
+) -> InMemoryListExecutor[HubRow]:
if prisma_client is None:
return InMemoryListExecutor(rows=rows, cells=_cells)
return InMemoryListExecutor(
diff --git a/litellm/types/proxy/management_endpoints/model_management_endpoints.py b/litellm/types/proxy/management_endpoints/model_management_endpoints.py
index b12745679df..9d9988ef39a 100644
--- a/litellm/types/proxy/management_endpoints/model_management_endpoints.py
+++ b/litellm/types/proxy/management_endpoints/model_management_endpoints.py
@@ -42,6 +42,8 @@ class ModelGroupInfoProxy(ModelGroupInfo):
health_status: str | None = Field(default=None)
health_response_time: float | None = Field(default=None)
health_checked_at: str | None = Field(default=None)
+ pass_through_path: str | None = Field(default=None)
+ pass_through_methods: tuple[str, ...] | None = Field(default=None)
custom_tags: tuple[str, ...] | None = None
custom_metadata: Mapping[str, str | int | float | bool] | None = None
diff --git a/litellm/ui/litellm-dashboard/src/components/PublicModelHubTableColumns.tsx b/litellm/ui/litellm-dashboard/src/components/PublicModelHubTableColumns.tsx
index eb89af22f17..a9f0947836b 100644
--- a/litellm/ui/litellm-dashboard/src/components/PublicModelHubTableColumns.tsx
+++ b/litellm/ui/litellm-dashboard/src/components/PublicModelHubTableColumns.tsx
@@ -30,6 +30,8 @@ export interface ModelGroupInfo {
health_status?: string;
health_response_time?: number;
health_checked_at?: string;
+ pass_through_path?: string | null;
+ pass_through_methods?: string[] | null;
description?: string;
custom_tags?: string[] | null;
custom_metadata?: Record {t("passThroughModelHub.description")} {t("addPassThrough.billing.description")}{t("passThroughModelHub.title")}
+ {t("addPassThrough.billing.title")}