From 382588a365f98ad63600fb72f2a3d91bd7af51e3 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 17:46:38 +0000 Subject: [PATCH] feat(proxy): publish opted-in pass-through endpoints on the Model Hub (#43905) * feat(proxy): publish opted-in pass-through endpoints on the Model Hub Pass-through endpoints gain display_name and show_in_model_hub. Endpoints that opt in appear on /public/v1/model_hub and /public/model_hub as rows in mode passthrough, named by their display name or their path, with the route in pass_through_path. The admin UI exposes both fields on the create and edit forms and the public hub renders passthrough rows with a route-specific curl example Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): carry pass-through methods onto hub rows and key dashboard rows by route Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): keep health, cost and shell quoting correct for pass-through hub rows Pass-through rows no longer inherit a router model's health check when they share its name, the public table shows n/a instead of Free for their token costs, and the generated curl example escapes shell characters in the route Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(ui): quote the method in the pass-through hub curl example A configured method string is placed after curl -X inside escaped double quotes so it cannot alter the copied command Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(ui): build pass-through hub fixtures through a helper Keeps the dashboard lint budget for inline object arguments under its limit Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): audit the pass-through Model Hub surface end to end Cover forwarding, pagination, validation, auth, updates, duplicate names, config-file precedence, health-name isolation, repeated reads and two chaos cells (create/delete churn, a killed proxy worker). Build the stored pass_through_endpoints list in one shot instead of appending to the list the config reader handed back Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): keep concurrent pass-through creates from overwriting each other Creating an endpoint appends to the general_settings list the proxy already holds instead of persisting a private copy, so two admins creating at the same time both land, as on main before this branch Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): serialize pass-through endpoint creates behind a process lock Each create copies the stored list under an asyncio lock before persisting, so simultaneous creates in one worker no longer overwrite each other and a rejected write (config-owned path) never lands in the in-memory settings Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): collect paginated pass-through rows without mutating a list Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(ui): let a pass-through with no stored timeout save from the edit form Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(ui): type the pass-through edit fixture so a null timeout compiles Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * perf(proxy): read published pass-throughs from the in-memory settings store The public Model Hub routes read the pass-through list this worker already serves from proxy_config.settings instead of querying the config table on every anonymous hub read. Also drops a mutable-ok suppression that no longer guards a violation Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): list a config file pass-through beside database rows on the hub Main merges database pass-throughs with config file ones on other paths since #43962, so a database write beside a config entry is accepted rather than rejected as config-owned Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy): type the monkeypatch fixture in the pass-through hub tests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(proxy): validate pass-through config entries instead of splatting unknown dicts Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(proxy): list model hub rows as a tagged union of model groups and pass-throughs Health enrichment and the no-router check now match on ModelGroupRow or PassThroughRow instead of testing pass_through_path in each route. Both hub routes share published_hub_rows, the v1 row helper is sync again, and the config list route goes back to main's inline merge since the hub no longer needs configured_pass_through_endpoints * test(ui): find the pass-through hub row by role to stay under the no-node-access budget * fix(proxy): keep database pass-through endpoints in the settings store across config reloads A config reload cleared the stored database row before rebuilding the pass-through list, so without store_model_in_db the list fell back to the config file entries alone. The Admin UI triggers that reload on page load through /get/config/callbacks, which dropped UI-created endpoints from the Model Hub until the next write * refactor(proxy): make the hub row matches exhaustive with assert_never * refactor(proxy): return the exhaustive match fallthrough and build hub test inputs in one shot Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(proxy): drop the in-process create lock from pass-through endpoints The lock only covered one worker, and the read-modify-write on the general_settings blob is a separate storage problem tracked as tech debt. Create goes back to main's behavior. * fix: align pass-through model hub with current lint rules Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test: share worker state for pass-through hub restart coverage Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix: show pass-through curl examples in model hub inspector Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix: skip invalid pass-through hub entries and await peer refresh Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: ryan Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: mateo <277851410+mateo-berri@users.noreply.github.com> --- litellm/proxy/_types.py | 8 + .../pass_through_endpoints.py | 6 +- litellm/proxy/proxy_server.py | 12 +- .../proxy/public_endpoints/model_hub_rows.py | 86 ++++ .../public_endpoints/public_endpoints.py | 83 ++-- .../public_endpoints/public_v1/model_hub.py | 78 +-- .../model_management_endpoints.py | 2 + .../components/PublicModelHubTableColumns.tsx | 29 +- .../add_pass_through.integration.test.tsx | 19 + .../src/components/add_pass_through.tsx | 45 ++ .../pass_through_info.integration.test.tsx | 44 +- .../src/components/pass_through_info.tsx | 52 +- .../publicModelHub/ModelInspector.tsx | 2 +- .../PublicAiHub.integration.test.tsx | 30 ++ .../publicModelHub/hubSnippets.test.ts | 41 +- .../components/publicModelHub/hubSnippets.ts | 22 + .../src/components/public_model_hub.test.tsx | 65 +++ .../src/components/public_model_hub.tsx | 102 ++-- .../src/i18n/locales/en.json | 12 + .../src/lib/http/schema.d.ts | 15 + .../management/test_pass_through_model_hub.py | 454 ++++++++++++++++++ .../proxy/proxy_server/test_proxy_config.py | 24 + .../public_v1/test_model_hub.py | 173 ++++++- 23 files changed, 1256 insertions(+), 148 deletions(-) create mode 100644 litellm/proxy/public_endpoints/model_hub_rows.py create mode 100644 tests/integration/management/test_pass_through_model_hub.py diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 804ad744989..d703c378ce8 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2348,6 +2348,14 @@ class PassThroughGenericEndpoint(LiteLLMPydanticObjectBase): default=None, description="List of HTTP methods this endpoint handles (e.g., ['GET', 'POST']). If None or empty, all methods (GET, POST, PUT, DELETE, PATCH) are supported for backward compatibility. This allows the same path to have different targets for different HTTP methods.", ) + display_name: str | None = Field( + default=None, + description="Human-readable name for this endpoint. The Model Hub lists the endpoint under this name, falling back to the path when unset.", + ) + show_in_model_hub: bool = Field( + default=False, + description="If True, this endpoint is published on the public Model Hub as a row in mode 'passthrough'.", + ) class PassThroughEndpointResponse(LiteLLMPydanticObjectBase): diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 8a728a7335a..b2caa4d837d 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -4748,7 +4748,7 @@ async def update_pass_through_endpoints( # Find the index for updating the list endpoint_index = None for idx, endpoint in enumerate(pass_through_endpoint_data): - _endpoint = PassThroughGenericEndpoint(**endpoint) if isinstance(endpoint, dict) else endpoint + _endpoint = PassThroughGenericEndpoint.model_validate(endpoint) if isinstance(endpoint, dict) else endpoint if _endpoint.id == endpoint_id: endpoint_index = idx break @@ -4981,7 +4981,7 @@ async def delete_pass_through_endpoints( # Find the index for deleting from the list endpoint_index = None for idx, endpoint in enumerate(pass_through_endpoint_data): - _endpoint = PassThroughGenericEndpoint(**endpoint) if isinstance(endpoint, dict) else endpoint + _endpoint = PassThroughGenericEndpoint.model_validate(endpoint) if isinstance(endpoint, dict) else endpoint if _endpoint.id == endpoint_id: endpoint_index = idx break @@ -5029,7 +5029,7 @@ def _find_endpoint_by_id( for endpoint in endpoints_data: _endpoint: PassThroughGenericEndpoint | None = None if isinstance(endpoint, dict): - _endpoint = PassThroughGenericEndpoint(**endpoint) + _endpoint = PassThroughGenericEndpoint.model_validate(endpoint) elif isinstance(endpoint, PassThroughGenericEndpoint): _endpoint = endpoint diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 7fd114f9838..e33966ceb29 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -5858,6 +5858,16 @@ def _reload_settings_store(section: Section, store: SettingsStore, section_confi store["pass_through_endpoints"] = serving_pass_throughs +def _clear_db_settings_beside_pass_through_endpoints(section: Section, store: SettingsStore) -> None: + stored_pass_throughs: Final = store.db_value("pass_through_endpoints") + store.apply_db_row( + section, + MappingProxyType({"pass_through_endpoints": stored_pass_throughs}) + if isinstance(stored_pass_throughs, list) + else _EMPTY_SETTINGS_MAPPING, + ) + + def _general_settings_row_is_readable(row: _ConfigParamValueRow | None) -> bool: return row is None or row.param_value is None or isinstance(row.param_value, Mapping) @@ -6708,7 +6718,7 @@ class ProxyConfig: ) else: for section, store in self._settings_stores.items(): - store.apply_db_row(section, _EMPTY_SETTINGS_MAPPING) + _clear_db_settings_beside_pass_through_endpoints(section, store) config = self._config_with_resolved_settings(config) diff --git a/litellm/proxy/public_endpoints/model_hub_rows.py b/litellm/proxy/public_endpoints/model_hub_rows.py new file mode 100644 index 00000000000..9d1d48ad1b8 --- /dev/null +++ b/litellm/proxy/public_endpoints/model_hub_rows.py @@ -0,0 +1,86 @@ +"""The rows the public Model Hub lists: published model groups and opted-in pass-through endpoints.""" + +from collections.abc import Iterator +from dataclasses import dataclass +from typing import Final, TypeAlias + +from pydantic import TypeAdapter, ValidationError + +import litellm +from litellm.proxy._types import PassThroughGenericEndpoint +from litellm.types.proxy.management_endpoints.model_management_endpoints import ModelGroupInfoProxy + +PASS_THROUGH_MODE: Final = "passthrough" +_SERVING_ENTRY: Final = TypeAdapter(dict[str, object]) + + +@dataclass(frozen=True, slots=True) +class ModelGroupRow: + info: ModelGroupInfoProxy + + +@dataclass(frozen=True, slots=True) +class PassThroughRow: + info: ModelGroupInfoProxy + + +HubRow: TypeAlias = ModelGroupRow | PassThroughRow + + +@dataclass(frozen=True, slots=True) +class NoModelsConfigured: + """Neither a router nor a published pass-through, so the hub has nothing to list.""" + + +def pass_through_row(endpoint: PassThroughGenericEndpoint) -> PassThroughRow: + return PassThroughRow( + info=ModelGroupInfoProxy( + model_group=endpoint.display_name or endpoint.path, + providers=[], + mode=PASS_THROUGH_MODE, + is_public_model_group=True, + pass_through_path=endpoint.path, + pass_through_methods=tuple(endpoint.methods) if endpoint.methods is not None else None, + ) + ) + + +def _serving_pass_through_endpoints() -> Iterator[PassThroughGenericEndpoint]: + """The endpoints this worker serves, from the in-memory general settings the config sync keeps current.""" + from litellm.proxy.proxy_server import proxy_config + + configured: Final = proxy_config.settings.get("pass_through_endpoints") + for entry in configured if isinstance(configured, list) else (): + try: + yield PassThroughGenericEndpoint.model_validate(_SERVING_ENTRY.validate_python(entry)) + except ValidationError: + continue + + +def _published_model_group_rows() -> tuple[ModelGroupRow, ...] | None: + """None when there is no router, which differs from a router that publishes nothing.""" + from litellm.proxy.proxy_server import ( + get_model_group_info, + llm_router, + ) + + if llm_router is None: + return None + if litellm.public_model_groups is None: + return () + return tuple( + ModelGroupRow(info=info) + for info in get_model_group_info( + llm_router=llm_router, all_models_str=litellm.public_model_groups, model_group=None + ) + ) + + +def published_hub_rows() -> tuple[HubRow, ...] | NoModelsConfigured: + pass_through_rows: Final = tuple( + pass_through_row(endpoint) for endpoint in _serving_pass_through_endpoints() if endpoint.show_in_model_hub + ) + model_group_rows: Final = _published_model_group_rows() + if model_group_rows is None and not pass_through_rows: + return NoModelsConfigured() + return (*(model_group_rows or ()), *pass_through_rows) diff --git a/litellm/proxy/public_endpoints/public_endpoints.py b/litellm/proxy/public_endpoints/public_endpoints.py index cb1e51d0deb..45abd3c4d5f 100644 --- a/litellm/proxy/public_endpoints/public_endpoints.py +++ b/litellm/proxy/public_endpoints/public_endpoints.py @@ -5,6 +5,7 @@ import re from collections.abc import Awaitable, Callable, Mapping, Sequence from functools import cache from importlib.resources import files +from types import MappingProxyType from typing import TYPE_CHECKING, Final, Protocol from fastapi import APIRouter, HTTPException, Request @@ -25,6 +26,13 @@ from litellm.proxy._types import ( ) from litellm.proxy.db.health_check_latest import latest_by_model_name from litellm.proxy.deps.repositories import PluginUploadRepositoryDep +from litellm.proxy.public_endpoints.model_hub_rows import ( + HubRow, + ModelGroupRow, + NoModelsConfigured, + PassThroughRow, + published_hub_rows, +) from litellm.proxy.utils import get_custom_url from litellm.repositories.plugin_publication import read_plugin_publisher, read_plugin_review from litellm.repositories.table_repositories import ClaudeCodePluginRepository @@ -276,45 +284,52 @@ def _agent_create_fields() -> tuple[AgentCreateInfo, ...]: response_model=list[ModelGroupInfoProxy], ) async def public_model_hub(): - import litellm - from litellm.proxy.health_endpoints._health_endpoints import ( - convert_health_check_to_dict, - ) - from litellm.proxy.proxy_server import ( - get_model_group_info, - llm_router, - prisma_client, - ) - - if llm_router is None: + rows: Final = published_hub_rows() + if isinstance(rows, NoModelsConfigured): raise HTTPException(status_code=400, detail=CommonProxyErrors.no_llm_router.value) - model_groups: list[ModelGroupInfoProxy] = [] - if litellm.public_model_groups is not None: - model_groups = get_model_group_info( # rebind-ok: pre-existing rebinding on a rename-only line - llm_router=llm_router, - all_models_str=litellm.public_model_groups, - model_group=None, + health_checks_map: Final = await _legacy_health_checks() + return [_with_legacy_health(row, health_checks_map) for row in rows] + + +async def _legacy_health_checks() -> Mapping[str, Mapping[str, object]]: + from litellm.proxy.proxy_server import prisma_client + + if prisma_client is None: + return MappingProxyType({}) + try: + latest_checks: Final = await prisma_client.get_all_latest_health_checks() + return MappingProxyType( + { + model_name: MappingProxyType( + { + "status": check.status, + "response_time_ms": check.response_time_ms, + "checked_at": check.checked_at.isoformat() if check.checked_at else None, + } + ) + for model_name, check in latest_by_model_name(latest_checks).items() + } ) + except Exception: + return MappingProxyType({}) - # Fetch health check information if available - health_checks_map: Final = {} - if prisma_client is not None: - try: - latest_checks: Final = await prisma_client.get_all_latest_health_checks() - for model_name, check in latest_by_model_name(latest_checks).items(): - health_checks_map[model_name] = convert_health_check_to_dict(check) - except Exception: - pass - for model_group in model_groups: - health_info = health_checks_map.get(model_group.model_group) - if health_info: - model_group.health_status = health_info.get("status") - model_group.health_response_time = health_info.get("response_time_ms") - model_group.health_checked_at = health_info.get("checked_at") - - return model_groups +def _with_legacy_health(row: HubRow, health_checks_map: Mapping[str, Mapping[str, object]]) -> ModelGroupInfoProxy: + match row: + case PassThroughRow(): + return row.info + case ModelGroupRow(): + health_info: Final = health_checks_map.get(row.info.model_group) + if not health_info: + return row.info + return row.info.model_copy( + update={ + "health_status": health_info.get("status"), + "health_response_time": health_info.get("response_time_ms"), + "health_checked_at": health_info.get("checked_at"), + } + ) @router.get( diff --git a/litellm/proxy/public_endpoints/public_v1/model_hub.py b/litellm/proxy/public_endpoints/public_v1/model_hub.py index e58149fba0b..a00138c94c9 100644 --- a/litellm/proxy/public_endpoints/public_v1/model_hub.py +++ b/litellm/proxy/public_endpoints/public_v1/model_hub.py @@ -8,7 +8,6 @@ from typing import Annotated, Final, Literal, Protocol from fastapi import APIRouter, Depends, Request from typing_extensions import ReadOnly, TypedDict -import litellm from litellm._logging import verbose_proxy_logger from litellm.proxy._types import CommonProxyErrors, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth @@ -24,6 +23,13 @@ from litellm.proxy.list_api.list_framework import ( handle_facet, handle_list, ) +from litellm.proxy.public_endpoints.model_hub_rows import ( + HubRow, + ModelGroupRow, + NoModelsConfigured, + PassThroughRow, + published_hub_rows, +) from litellm.proxy.utils import PrismaClient from litellm.types.proxy.management_endpoints.management_v1 import ( FacetListResponse, @@ -93,9 +99,19 @@ class HealthEnricher: lookup: HealthSnapshotLookup - async def __call__(self, rows: Sequence[ModelGroupInfoProxy]) -> Sequence[ModelGroupInfoProxy]: - health: Final = await self.lookup.latest_for(tuple(row.model_group for row in rows)) - return tuple(_with_health(row, health.get(row.model_group)) for row in rows) + async def __call__(self, rows: Sequence[HubRow]) -> Sequence[HubRow]: + health: Final = await self.lookup.latest_for( + tuple(row.info.model_group for row in rows if isinstance(row, ModelGroupRow)) + ) + return tuple(_row_with_health(row, health) for row in rows) + + +def _row_with_health(row: HubRow, health: Mapping[str, HealthSnapshot]) -> HubRow: + match row: + case PassThroughRow(): + return row + case ModelGroupRow(): + return ModelGroupRow(info=_with_health(row.info, health.get(row.info.model_group))) FEATURE_PREFIX: Final = "supports_" @@ -120,26 +136,26 @@ def _enabled_features(fields: Mapping[str, object]) -> tuple[str, ...]: ) -def _cells(row: ModelGroupInfoProxy) -> Cells: +def _cells(row: HubRow) -> Cells: + info: Final = row.info return MappingProxyType( { - "model_group": row.model_group, - "mode": row.mode, - "providers": tuple(row.providers), - "features": _features(row), - "max_input_tokens": row.max_input_tokens, - "max_output_tokens": row.max_output_tokens, - "input_cost_per_token": row.input_cost_per_token, - "output_cost_per_token": row.output_cost_per_token, - "rpm": row.rpm, - "tpm": row.tpm, + "model_group": info.model_group, + "mode": info.mode, + "providers": tuple(info.providers), + "features": _features(info), + "max_input_tokens": info.max_input_tokens, + "max_output_tokens": info.max_output_tokens, + "input_cost_per_token": info.input_cost_per_token, + "output_cost_per_token": info.output_cost_per_token, + "rpm": info.rpm, + "tpm": info.tpm, } ) -def _serialize(row: ModelGroupInfoProxy) -> ModelGroupInfoProxy: - """The row shape is the wire shape: the rows served are the router's own model group records.""" - return row +def _serialize(row: HubRow) -> ModelGroupInfoProxy: + return row.info def _scope(_caller: UserAPIKeyAuth) -> Scope: @@ -163,7 +179,7 @@ MODEL_HUB_FACETS: Final[Mapping[str, str]] = MappingProxyType( {"providers": "providers", "modes": "mode", "features": "features"} ) -MODEL_HUB_LIST_SPEC: Final[ListSpec[ModelGroupInfoProxy, ModelGroupInfoProxy]] = ListSpec( +MODEL_HUB_LIST_SPEC: Final[ListSpec[HubRow, ModelGroupInfoProxy]] = ListSpec( resource="model groups", sortable=frozenset( ( @@ -189,13 +205,9 @@ MODEL_HUB_LIST_SPEC: Final[ListSpec[ModelGroupInfoProxy, ModelGroupInfoProxy]] = ) -def _published_rows() -> Sequence[ModelGroupInfoProxy]: - from litellm.proxy.proxy_server import ( - get_model_group_info, # pyright: ignore[reportPrivateUsage] # /public/model_hub imports it the same way - llm_router, - ) - - if llm_router is None: +def _published_rows() -> Sequence[HubRow]: + rows: Final = published_hub_rows() + if isinstance(rows, NoModelsConfigured): raise ManagementProblem( ProblemDetail( type=f"{PROBLEM_TYPE_BASE}no-llm-router", @@ -204,21 +216,13 @@ def _published_rows() -> Sequence[ModelGroupInfoProxy]: detail=CommonProxyErrors.no_llm_router.value, ) ) - if litellm.public_model_groups is None: - return () - return tuple( - get_model_group_info( - llm_router=llm_router, - all_models_str=litellm.public_model_groups, - model_group=None, - ) - ) + return rows def _executor( - rows: Sequence[ModelGroupInfoProxy], + rows: Sequence[HubRow], prisma_client: PrismaClient | None, -) -> InMemoryListExecutor[ModelGroupInfoProxy]: +) -> InMemoryListExecutor[HubRow]: if prisma_client is None: return InMemoryListExecutor(rows=rows, cells=_cells) return InMemoryListExecutor( diff --git a/litellm/types/proxy/management_endpoints/model_management_endpoints.py b/litellm/types/proxy/management_endpoints/model_management_endpoints.py index b12745679df..9d9988ef39a 100644 --- a/litellm/types/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/types/proxy/management_endpoints/model_management_endpoints.py @@ -42,6 +42,8 @@ class ModelGroupInfoProxy(ModelGroupInfo): health_status: str | None = Field(default=None) health_response_time: float | None = Field(default=None) health_checked_at: str | None = Field(default=None) + pass_through_path: str | None = Field(default=None) + pass_through_methods: tuple[str, ...] | None = Field(default=None) custom_tags: tuple[str, ...] | None = None custom_metadata: Mapping[str, str | int | float | bool] | None = None diff --git a/litellm/ui/litellm-dashboard/src/components/PublicModelHubTableColumns.tsx b/litellm/ui/litellm-dashboard/src/components/PublicModelHubTableColumns.tsx index eb89af22f17..a9f0947836b 100644 --- a/litellm/ui/litellm-dashboard/src/components/PublicModelHubTableColumns.tsx +++ b/litellm/ui/litellm-dashboard/src/components/PublicModelHubTableColumns.tsx @@ -30,6 +30,8 @@ export interface ModelGroupInfo { health_status?: string; health_response_time?: number; health_checked_at?: string; + pass_through_path?: string | null; + pass_through_methods?: string[] | null; description?: string; custom_tags?: string[] | null; custom_metadata?: Record | null; @@ -90,6 +92,11 @@ const formatCapabilityName = (key: string) => const formatCost = (cost: number) => `$${(cost * 1_000_000).toFixed(4)}`; +const tokenCostLabel = (model: ModelGroupInfo, cost: number | null | undefined): string => { + if (model.pass_through_path) return "n/a"; + return cost ? formatCost(cost) : i18n.t("aiHub.publicModelHubColumns.free"); +}; + const formatTokens = (tokens: number | undefined) => { if (!tokens) return i18n.t("aiHub.publicModelHubColumns.notAvailable"); if (tokens >= 1000) return `${(tokens / 1000).toFixed(0)}K`; @@ -112,6 +119,8 @@ const getModeIcon = (mode: string) => { return "🔄"; case "embedding": return "📄"; + case "passthrough": + return "🔀"; default: return "🤖"; } @@ -304,25 +313,17 @@ export const getPublicModelHubColumns = ({
{i18n.t("aiHub.publicModelHubColumns.inputCost", { - cost: row.original.input_cost_per_token - ? formatCost(row.original.input_cost_per_token) - : i18n.t("aiHub.publicModelHubColumns.free"), + cost: tokenCostLabel(row.original, row.original.input_cost_per_token), })} {i18n.t("aiHub.publicModelHubColumns.outputCost", { - cost: row.original.output_cost_per_token - ? formatCost(row.original.output_cost_per_token) - : i18n.t("aiHub.publicModelHubColumns.free"), + cost: tokenCostLabel(row.original, row.original.output_cost_per_token), })}
) : ( - - {row.original.input_cost_per_token - ? formatCost(row.original.input_cost_per_token) - : i18n.t("aiHub.publicModelHubColumns.free")} - + {tokenCostLabel(row.original, row.original.input_cost_per_token)} ), }, { @@ -334,11 +335,7 @@ export const getPublicModelHubColumns = ({ ), size: 110, cell: ({ row }) => ( - - {row.original.output_cost_per_token - ? formatCost(row.original.output_cost_per_token) - : i18n.t("aiHub.publicModelHubColumns.free")} - + {tokenCostLabel(row.original, row.original.output_cost_per_token)} ), }, { diff --git a/litellm/ui/litellm-dashboard/src/components/add_pass_through.integration.test.tsx b/litellm/ui/litellm-dashboard/src/components/add_pass_through.integration.test.tsx index a970922d176..ac82e6abb67 100644 --- a/litellm/ui/litellm-dashboard/src/components/add_pass_through.integration.test.tsx +++ b/litellm/ui/litellm-dashboard/src/components/add_pass_through.integration.test.tsx @@ -80,6 +80,8 @@ const reopenedPayload = { auth: true, timeout: undefined, cost_per_request: undefined, + display_name: undefined, + show_in_model_hub: false, }; describe("add_pass_through submit payload", () => { @@ -112,9 +114,26 @@ describe("add_pass_through submit payload", () => { auth: true, timeout: "900", cost_per_request: "1.5", + display_name: undefined, + show_in_model_hub: false, }); }); + it("sends the display name and the Model Hub opt-in when they are set", async () => { + const user = setup(); + renderForm(); + await openModal(user); + await fillRequiredFields(user); + fireEvent.change(screen.getByPlaceholderText("Clinical NER"), { target: { value: "Clinical NER" } }); + await user.click(screen.getByRole("switch", { name: "Show on Model Hub" })); + + await submit(user); + + await waitFor(() => expect(createPassThroughEndpoint).toHaveBeenCalled()); + expect(lastPayload().display_name).toBe("Clinical NER"); + expect(lastPayload().show_in_model_hub).toBe(true); + }); + it("submits numeric fields as strings, not numbers", async () => { const user = setup(); renderForm(); diff --git a/litellm/ui/litellm-dashboard/src/components/add_pass_through.tsx b/litellm/ui/litellm-dashboard/src/components/add_pass_through.tsx index ce4685d47df..31d6e28a467 100644 --- a/litellm/ui/litellm-dashboard/src/components/add_pass_through.tsx +++ b/litellm/ui/litellm-dashboard/src/components/add_pass_through.tsx @@ -52,6 +52,8 @@ const passThroughFormSchema = z.object({ auth: z.boolean(), timeout: z.string().optional(), cost_per_request: z.string().optional(), + display_name: z.string().optional(), + show_in_model_hub: z.boolean().optional(), }); type PassThroughFormValues = z.output; @@ -66,6 +68,8 @@ const emptyFormValues = { auth: true, timeout: undefined, cost_per_request: undefined, + display_name: undefined, + show_in_model_hub: false, } as unknown as PassThroughFormValues; const labelWithHint = (label: React.ReactNode, hint: string): React.ReactNode => ( @@ -129,6 +133,8 @@ const AddPassThroughEndpoint: React.FC = ({ auth: values.auth, timeout: values.timeout, cost_per_request: values.cost_per_request, + display_name: optionalText(values.display_name ?? ""), + show_in_model_hub: values.show_in_model_hub ?? false, ...(Object.keys(guardrails).length > 0 ? { guardrails } : {}), }; @@ -336,6 +342,45 @@ const AddPassThroughEndpoint: React.FC = ({ + +

{t("passThroughModelHub.title")}

+

{t("passThroughModelHub.description")}

+ +
+ + {({ value, ...field }) => ( + + )} + + +
+
+
{t("passThroughModelHub.show")}
+
{t("passThroughModelHub.showHint")}
+
+ + {({ value, onChange, ref: _ref, ...field }) => ( + + )} + +
+
+ +

{t("addPassThrough.billing.title")}

{t("addPassThrough.billing.description")}

diff --git a/litellm/ui/litellm-dashboard/src/components/pass_through_info.integration.test.tsx b/litellm/ui/litellm-dashboard/src/components/pass_through_info.integration.test.tsx index 051195c72b4..ea7d71474c1 100644 --- a/litellm/ui/litellm-dashboard/src/components/pass_through_info.integration.test.tsx +++ b/litellm/ui/litellm-dashboard/src/components/pass_through_info.integration.test.tsx @@ -1,6 +1,7 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; +import type { ComponentProps } from "react"; import userEvent, { PointerEventsCheckLevel } from "@testing-library/user-event"; -import { renderWithProviders, screen, waitFor } from "../../tests/test-utils"; +import { fireEvent, renderWithProviders, screen, waitFor } from "../../tests/test-utils"; import PassThroughInfoView from "./pass_through_info"; const updatePassThroughEndpoint = vi.fn(); @@ -31,7 +32,9 @@ const endpoint = { methods: ["GET"], }; -const renderView = (data = endpoint) => +type EndpointData = ComponentProps["endpointData"]; + +const renderView = (data: EndpointData = endpoint) => renderWithProviders(); const openEditForm = async (user: User) => { @@ -72,9 +75,34 @@ describe("pass_through_info update payload", () => { auth: false, methods: ["GET"], guardrails: undefined, + display_name: "", + show_in_model_hub: false, }); }); + it("sends the edited display name and Model Hub opt-in", async () => { + const user = setup(); + renderWithProviders( + , + ); + await openEditForm(user); + expect(screen.getByLabelText("Display Name")).toHaveValue("Clinical NER"); + expect(screen.getByRole("switch", { name: "Show on Model Hub" })).toBeChecked(); + fireEvent.change(screen.getByLabelText("Display Name"), { target: { value: " Clinical NER v2 " } }); + await user.click(screen.getByRole("switch", { name: "Show on Model Hub" })); + + await save(user); + + await waitFor(() => expect(updatePassThroughEndpoint).toHaveBeenCalled()); + expect(lastPayload().display_name).toBe("Clinical NER v2"); + expect(lastPayload().show_in_model_hub).toBe(false); + }); + it("submits numeric fields as numbers, not strings", async () => { const user = setup(); renderView(); @@ -87,6 +115,18 @@ describe("pass_through_info update payload", () => { expect(lastPayload().timeout).toBe(600); }); + it("saves an endpoint whose stored timeout is null without a validation error", async () => { + const user = setup(); + renderView({ ...endpoint, timeout: null }); + await openEditForm(user); + + await save(user); + + await waitFor(() => expect(updatePassThroughEndpoint).toHaveBeenCalled()); + expect(screen.queryByText("Invalid input")).not.toBeInTheDocument(); + expect(lastPayload().timeout).toBeUndefined(); + }); + it("keeps the path from the loaded endpoint rather than the form", async () => { const user = setup(); renderView(); diff --git a/litellm/ui/litellm-dashboard/src/components/pass_through_info.tsx b/litellm/ui/litellm-dashboard/src/components/pass_through_info.tsx index 9852892e833..818e0d71d9b 100644 --- a/litellm/ui/litellm-dashboard/src/components/pass_through_info.tsx +++ b/litellm/ui/litellm-dashboard/src/components/pass_through_info.tsx @@ -33,6 +33,8 @@ const endpointSettingsSchema = z.object({ cost_per_request: z.number().optional(), timeout: z.number().optional(), auth: z.boolean(), + display_name: z.string().optional(), + show_in_model_hub: z.boolean(), }); type EndpointSettingsValues = z.output; @@ -104,10 +106,12 @@ interface PassThroughEndpoint { headers: Record; include_subpath?: boolean; cost_per_request?: number; - timeout?: number; + timeout?: number | null; auth?: boolean; methods?: string[]; guardrails?: Record; + display_name?: string | null; + show_in_model_hub?: boolean; } // Password field component for headers @@ -159,8 +163,10 @@ const PassThroughInfoView: React.FC = ({ methods: initialEndpointData.methods || [], include_subpath: initialEndpointData.include_subpath || false, cost_per_request: initialEndpointData.cost_per_request, - timeout: initialEndpointData.timeout, + timeout: initialEndpointData.timeout ?? undefined, auth: initialEndpointData.auth ?? true, + display_name: initialEndpointData.display_name ?? undefined, + show_in_model_hub: initialEndpointData.show_in_model_hub ?? false, }, }); @@ -195,6 +201,8 @@ const PassThroughInfoView: React.FC = ({ auth: values.auth, methods: values.methods.length > 0 ? values.methods : undefined, guardrails: guardrails && Object.keys(guardrails).length > 0 ? guardrails : undefined, + display_name: values.display_name?.trim() ?? "", + show_in_model_hub: values.show_in_model_hub, }; await updatePassThroughEndpoint(accessToken, endpointData.id, updateData); @@ -323,6 +331,11 @@ const PassThroughInfoView: React.FC = ({

)} + {endpointData.show_in_model_hub && ( +
+ {t("passThroughModelHub.published")} +
+ )} @@ -422,6 +435,27 @@ const PassThroughInfoView: React.FC = ({ )} + + {({ value, ...field }) => ( + + )} + + + + {({ value, onChange, ref: _ref, ...field }) => ( + + )} + + {({ value, ...field }) => (