diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 3da30070b9e..34023245ee8 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2551,6 +2551,18 @@ class PluginConfig(LiteLLMPydanticObjectBase): ) +class AdvertisedModel(LiteLLMPydanticObjectBase): + """A catalog-only entry listed by /v1/models without a routable deployment. + + Discovery only: nothing here registers a route, so a request naming this id + still fails as an unknown model. Use it to advertise something the proxy + does not serve itself, e.g. a realtime endpoint clients connect to directly. + """ + + id: str = Field(description="model id as clients see it in the listing") + owned_by: str = Field(description="owner reported for this entry, e.g. the provider name") + + class CoordinationRedisNode(LiteLLMPydanticObjectBase): """A single startup node of a cluster-mode Redis used for proxy coordination.""" @@ -2644,6 +2656,11 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): plugins: list[PluginConfig] | None = Field( None, description="external services registered as embeddable UI plugins" ) + advertised_models: list[AdvertisedModel] | None = Field( + None, + description="catalog-only entries added to /v1/models for discovery; they register no route, so " + "GET /v1/models/{id} still reports them as not found and a request naming one fails as an unknown model", + ) key_management_system: KeyManagementSystem | None = Field( None, description="key manager to load keys from / decrypt keys with" ) diff --git a/litellm/proxy/common_utils/advertised_models.py b/litellm/proxy/common_utils/advertised_models.py new file mode 100644 index 00000000000..295fa1da09d --- /dev/null +++ b/litellm/proxy/common_utils/advertised_models.py @@ -0,0 +1,88 @@ +"""Catalog-only entries appended to the model listing. + +`general_settings.advertised_models` lets an operator advertise a model id on +`GET /v1/models` without registering a routable deployment for it. The use case +is a model clients reach on their own, e.g. a realtime endpoint over a +websocket: it belongs in the client's model picker, but the proxy never serves +it, and inventing a placeholder deployment just to make it appear would leave a +live route that fails confusingly when someone calls it. + +Discovery only, in both directions. Nothing here registers a route, so a request +naming a catalog id still fails as an unknown model, and `GET /v1/models/{id}` +still answers 404: the catalog says what exists, not what this proxy serves. + +A catalog entry never displaces a routed one. An id already in the listing wins, +so a typo here cannot mask a working model or impersonate it, and a repeated id +is listed once. + +A row carries exactly what the operator declared. Nothing is inferred from the +cost map, so an entry that happens to share a name with a known model does not +silently pick up that model's mode or token limits, and an entry the cost map +cannot resolve does not drive `get_llm_provider` into printing its provider list +to stdout on every listing request. + +Misconfiguration fails open: an `advertised_models` value that does not validate +is logged and skipped, leaving the listing exactly as it would have been. +""" + +from __future__ import annotations + +from collections.abc import Collection, Mapping +from typing import Final + +from pydantic import TypeAdapter, ValidationError + +from litellm._logging import verbose_proxy_logger +from litellm.constants import DEFAULT_MODEL_CREATED_AT_TIME +from litellm.proxy._types import AdvertisedModel +from litellm.types.proxy.model_listing import ModelInfoResponse + +ADVERTISED_MODELS_SETTING: Final = "advertised_models" + +_ADVERTISED_MODELS_ADAPTER: Final = TypeAdapter(tuple[AdvertisedModel, ...]) + + +def configured_advertised_models(general_settings: Mapping[str, object]) -> tuple[AdvertisedModel, ...]: + """Typed `advertised_models` entries, or none when unset or malformed.""" + configured: Final = general_settings.get(ADVERTISED_MODELS_SETTING) + if configured is None: + return () + try: + return _ADVERTISED_MODELS_ADAPTER.validate_python(configured) + except ValidationError as validation_error: + verbose_proxy_logger.warning( + "general_settings.%s is not a list of {id, owned_by} entries, skipping it: %s", + ADVERTISED_MODELS_SETTING, + validation_error, + ) + return () + + +def _listing_row(entry: AdvertisedModel) -> ModelInfoResponse: + row: Final[ModelInfoResponse] = { + "id": entry.id, + "object": "model", + "created": DEFAULT_MODEL_CREATED_AT_TIME, + "owned_by": entry.owned_by, + } + return row + + +def advertised_model_rows( + general_settings: Mapping[str, object], + listed_ids: Collection[str], +) -> tuple[ModelInfoResponse, ...]: + """Listing rows for the configured catalog entries, in configured order. + + `listed_ids` are the ids the listing already carries; entries naming one of + them are dropped so a routed model is never displaced. + """ + already_listed: Final = frozenset(listed_ids) + candidates: Final = tuple( + entry for entry in configured_advertised_models(general_settings) if entry.id not in already_listed + ) + return tuple( + _listing_row(entry) + for index, entry in enumerate(candidates) + if all(earlier.id != entry.id for earlier in candidates[:index]) + ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index ed4ea347c2e..c6e2348726a 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -394,6 +394,7 @@ from litellm.proxy.common_request_processing import ( resolve_litellm_call_id, ttft_keepalive_interval, ) +from litellm.proxy.common_utils.advertised_models import advertised_model_rows from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import ( AuthCacheInvalidationSubscriber, ) @@ -11260,6 +11261,13 @@ async def model_list( configured (cooldown remains the sole exclusion mechanism). Hiding is presentation-only: a hidden model can still be called directly. + + Set `general_settings.advertised_models` to add catalog-only entries, each + an `{id, owned_by}` pair, for models this proxy does not serve itself (say a + realtime endpoint clients connect to directly). They are listed for + discovery only and register no route, so a request naming one fails as an + unknown model and `GET /v1/models/{id}` reports it as not found. An entry + whose id is already listed is dropped, so a routed model is never displaced. """ global llm_model_list, general_settings, llm_router, prisma_client, user_api_key_cache, proxy_logging_obj @@ -11383,6 +11391,12 @@ async def model_list( model_info["id"] = response_id model_data.append(model_info) + # Catalog-only entries are advertised to every caller: they name no deployment, + # so the per-caller filters above have nothing to scope them by. A listing asked + # for access groups alone gets none of them, since a catalog entry is not a group. + if not only_model_access_groups: + model_data.extend(advertised_model_rows(settings, tuple(row["id"] for row in model_data))) + if wants_anthropic_format: admin_listing: Final = cast(Sequence[ModelInfoResponse], model_data) # cast-ok: rows built above return create_anthropic_model_list_response( @@ -11443,6 +11457,10 @@ async def model_list( model_info["id"] = response_id model_data.append(model_info) + # Same catalog merge as the scope=expand branch above. + if not only_model_access_groups: + model_data.extend(advertised_model_rows(settings, tuple(row["id"] for row in model_data))) + if wants_anthropic_format: listing: Final = cast(Sequence[ModelInfoResponse], model_data) # cast-ok: rows built above return create_anthropic_model_list_response( diff --git a/tests/test_litellm/proxy/common_utils/test_advertised_models.py b/tests/test_litellm/proxy/common_utils/test_advertised_models.py new file mode 100644 index 00000000000..7495432928f --- /dev/null +++ b/tests/test_litellm/proxy/common_utils/test_advertised_models.py @@ -0,0 +1,93 @@ +"""Tests for `general_settings.advertised_models`, the catalog-only entries +appended to the model listing by `litellm.proxy.common_utils.advertised_models`. +""" + +from typing import Final + +import litellm +from litellm.constants import DEFAULT_MODEL_CREATED_AT_TIME +from litellm.proxy.common_utils.advertised_models import ( + advertised_model_rows, + configured_advertised_models, +) + +CATALOG_ENTRY: Final = {"id": "catalog-only-model", "owned_by": "example-provider"} + + +def _settings(*entries: object) -> dict[str, object]: + return {"advertised_models": list(entries)} + + +def test_entry_becomes_a_row_carrying_its_configured_id_and_owner(): + rows: Final = advertised_model_rows(_settings(CATALOG_ENTRY), []) + + assert rows == ( + { + "id": "catalog-only-model", + "object": "model", + "created": DEFAULT_MODEL_CREATED_AT_TIME, + "owned_by": "example-provider", + }, + ), f"expected one row carrying exactly the configured id and owner, got {rows}" + + +def test_entry_naming_an_already_listed_model_is_dropped(): + """A routed model is never displaced by a catalog entry that shares its id.""" + rows: Final = advertised_model_rows(_settings({"id": "routed-model", "owned_by": "impostor"}), ["routed-model"]) + + assert rows == (), f"catalog entry must not shadow the routed model already listed, got {rows}" + + +def test_repeated_id_is_listed_once_keeping_the_first_entry(): + rows: Final = advertised_model_rows( + _settings(CATALOG_ENTRY, {"id": "catalog-only-model", "owned_by": "second-declaration"}), + [], + ) + + assert [row["owned_by"] for row in rows] == ["example-provider"], ( + f"a repeated id should be listed once, keeping the first declaration, got {rows}" + ) + + +def test_no_setting_produces_no_rows(): + assert advertised_model_rows({}, []) == (), "an unset advertised_models must add nothing to the listing" + + +def test_malformed_setting_is_skipped_rather_than_raising(): + """Misconfiguration fails open: the listing is returned as if nothing was set.""" + missing_owner: Final = advertised_model_rows({"advertised_models": [{"id": "no-owner"}]}, []) + not_a_list: Final = advertised_model_rows({"advertised_models": "catalog-only-model"}, []) + + assert missing_owner == (), f"an entry without owned_by must be skipped, got {missing_owner}" + assert not_a_list == (), f"a non-list advertised_models must be skipped, got {not_a_list}" + + +def test_row_carries_only_declared_fields_even_for_a_model_the_cost_map_knows(): + """Catalog rows are exactly what the operator declared, never enriched. + + The id is taken from the live cost map rather than hard-coded, so this keeps + testing the no-enrichment guarantee as the catalog changes. + """ + known_model: Final = next(iter(litellm.model_cost)) + + rows: Final = advertised_model_rows(_settings({"id": known_model, "owned_by": "example-provider"}), []) + + assert rows == ( + { + "id": known_model, + "object": "model", + "created": DEFAULT_MODEL_CREATED_AT_TIME, + "owned_by": "example-provider", + }, + ), f"a catalog row must not pick up cost map details for {known_model}, got {rows}" + + +def test_configured_entries_are_typed_and_keep_their_order(): + entries: Final = configured_advertised_models( + _settings(CATALOG_ENTRY, {"id": "second", "owned_by": "other-provider"}) + ) + + assert [(entry.id, entry.owned_by) for entry in entries] == [ + ("catalog-only-model", "example-provider"), + ("second", "other-provider"), + ], f"entries should be parsed in configured order, got {entries}" diff --git a/tests/test_litellm/proxy/test_model_list_advertised_models.py b/tests/test_litellm/proxy/test_model_list_advertised_models.py new file mode 100644 index 00000000000..6bcf661b793 --- /dev/null +++ b/tests/test_litellm/proxy/test_model_list_advertised_models.py @@ -0,0 +1,132 @@ +"""Tests for catalog-only entries on GET /v1/models. + +`general_settings.advertised_models` adds entries to the listing for models the +proxy does not serve itself. They are discovery only: no deployment is +registered, so routing is untouched and a routed model is never displaced. +""" + +from typing import Final +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from litellm.proxy import proxy_server +from litellm.proxy._types import UserAPIKeyAuth + +CATALOG_SETTINGS: Final = {"advertised_models": [{"id": "catalog-only-model", "owned_by": "example-provider"}]} + + +@pytest.fixture +def patched_model_list(monkeypatch): + """Stub router + utility helpers used by `model_list`.""" + from litellm.proxy import utils as proxy_utils + + router: Final = MagicMock() + router.get_fully_blocked_model_names = MagicMock(return_value=set()) + router.async_get_fully_unhealthy_model_names = AsyncMock(return_value=set()) + router.get_model_names = MagicMock(return_value=["routed-model"]) + router.get_model_access_groups = MagicMock(return_value={}) + + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "user_model", None) + monkeypatch.setattr(proxy_server, "general_settings", {}) + + async def _fake_get_available_models_for_user(**kwargs): + return ["routed-model"] + + monkeypatch.setattr(proxy_utils, "get_available_models_for_user", _fake_get_available_models_for_user) + + def _fake_create_model_info_response(model_id, provider="openai", **kwargs): + return {"id": model_id, "object": "model", "created": 0, "owned_by": provider} + + monkeypatch.setattr(proxy_utils, "create_model_info_response", _fake_create_model_info_response) + + return router + + +async def _listing(**kwargs): + response: Final = await proxy_server.model_list(user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"), **kwargs) + return response["data"] + + +@pytest.mark.asyncio +async def test_catalog_entry_is_listed_alongside_routed_models(patched_model_list, monkeypatch): + monkeypatch.setattr(proxy_server, "general_settings", CATALOG_SETTINGS) + + rows: Final = await _listing() + + assert [row["id"] for row in rows] == [ + "routed-model", + "catalog-only-model", + ], f"the catalog entry should be listed after the routed models, got {rows}" + assert rows[1]["owned_by"] == "example-provider", f"the configured owner should be reported, got {rows[1]}" + + +@pytest.mark.asyncio +async def test_routed_rows_are_untouched_by_the_catalog(patched_model_list, monkeypatch): + """The routed model's row is identical with and without catalog entries.""" + without_catalog: Final = await _listing() + + monkeypatch.setattr(proxy_server, "general_settings", CATALOG_SETTINGS) + with_catalog: Final = await _listing() + + assert with_catalog[0] == without_catalog[0], ( + "configuring advertised_models must not change an existing row: " + f"{with_catalog[0]} differs from {without_catalog[0]}" + ) + + +@pytest.mark.asyncio +async def test_listing_is_unchanged_when_no_catalog_is_configured(patched_model_list): + rows: Final = await _listing() + + assert rows == [{"id": "routed-model", "object": "model", "created": 0, "owned_by": "openai"}], ( + f"with no advertised_models the listing must be exactly what it was before, got {rows}" + ) + + +@pytest.mark.asyncio +async def test_catalog_entry_cannot_displace_a_routed_model_of_the_same_id(patched_model_list, monkeypatch): + monkeypatch.setattr( + proxy_server, + "general_settings", + {"advertised_models": [{"id": "routed-model", "owned_by": "impostor"}]}, + ) + + rows: Final = await _listing() + + assert rows == [{"id": "routed-model", "object": "model", "created": 0, "owned_by": "openai"}], ( + f"the routed model must be listed once, unchanged, got {rows}" + ) + + +@pytest.mark.asyncio +async def test_access_group_only_listing_carries_no_catalog_entries(patched_model_list, monkeypatch): + """A caller asking for access groups alone gets none: a catalog entry is not a group.""" + monkeypatch.setattr(proxy_server, "general_settings", CATALOG_SETTINGS) + + rows: Final = await _listing(only_model_access_groups=True) + + assert "catalog-only-model" not in [row["id"] for row in rows], ( + f"only_model_access_groups must not surface catalog entries, got {rows}" + ) + + +@pytest.mark.asyncio +async def test_catalog_entry_is_listed_for_scope_expand(patched_model_list, monkeypatch): + from litellm.proxy.auth import model_checks + from litellm.proxy.management_endpoints import common_utils + + async def _fake_admin(**kwargs): + return True + + monkeypatch.setattr(common_utils, "_user_has_admin_privileges", _fake_admin) + monkeypatch.setattr(model_checks, "get_complete_model_list", lambda **kwargs: ["routed-model"]) + monkeypatch.setattr(proxy_server, "general_settings", CATALOG_SETTINGS) + + rows: Final = await _listing(scope="expand") + + assert [row["id"] for row in rows] == [ + "routed-model", + "catalog-only-model", + ], f"the admin listing should carry catalog entries too, got {rows}" diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index bba67bcf6c2..af810a1b0a5 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -9798,6 +9798,13 @@ export interface paths { * configured (cooldown remains the sole exclusion mechanism). * Hiding is presentation-only: a hidden model can still be * called directly. + * + * Set `general_settings.advertised_models` to add catalog-only entries, each + * an `{id, owned_by}` pair, for models this proxy does not serve itself (say a + * realtime endpoint clients connect to directly). They are listed for + * discovery only and register no route, so a request naming one fails as an + * unknown model and `GET /v1/models/{id}` reports it as not found. An entry + * whose id is already listed is dropped, so a routed model is never displaced. */ get: operations["model_list_models_get"]; put?: never; @@ -20104,6 +20111,13 @@ export interface paths { * configured (cooldown remains the sole exclusion mechanism). * Hiding is presentation-only: a hidden model can still be * called directly. + * + * Set `general_settings.advertised_models` to add catalog-only entries, each + * an `{id, owned_by}` pair, for models this proxy does not serve itself (say a + * realtime endpoint clients connect to directly). They are listed for + * discovery only and register no route, so a request naming one fails as an + * unknown model and `GET /v1/models/{id}` reports it as not found. An entry + * whose id is already listed is dropped, so a routed model is never displaced. */ get: operations["model_list_v1_models_get"]; put?: never; @@ -24128,6 +24142,26 @@ export interface components { [key: string]: string; }; }; + /** + * AdvertisedModel + * @description A catalog-only entry listed by /v1/models without a routable deployment. + * + * Discovery only: nothing here registers a route, so a request naming this id + * still fails as an unknown model. Use it to advertise something the proxy + * does not serve itself, e.g. a realtime endpoint clients connect to directly. + */ + AdvertisedModel: { + /** + * Id + * @description model id as clients see it in the listing + */ + id: string; + /** + * Owned By + * @description owner reported for this entry, e.g. the provider name + */ + owned_by: string; + }; /** * AgentCapabilities * @description Defines optional capabilities supported by an agent. @@ -28013,6 +28047,11 @@ export interface components { * @default 1 */ admission_queue_timeout_seconds: number; + /** + * Advertised Models + * @description catalog-only entries added to /v1/models for discovery; they register no route, so GET /v1/models/{id} still reports them as not found and a request naming one fails as an unknown model + */ + advertised_models?: components["schemas"]["AdvertisedModel"][] | null; /** * Alert To Webhook Url * @description Mapping of alert type to webhook url. e.g. `alert_to_webhook_url: {'budget_alerts': 'https://nothooks.slack.com/services/T00000000/B00000000/XXXXXXXXXXXXXXXXXXXXXXXX'}`