diff --git a/litellm/router.py b/litellm/router.py index 1a3d1f54e29..df23a3d9e5b 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -285,6 +285,7 @@ from litellm.router_utils.embedding_batch_size_check import ( from litellm.router_utils.endpoint_eligibility import ( eligible_deployments, endpoint_selection, + warn_on_unenforced_endpoint_flag, ) from litellm.router_utils.fallback_event_handlers import ( MID_STREAM_FALLBACK_CONTROLS_KEY, @@ -10625,6 +10626,7 @@ class Router: declared_params: Final = _LITELLM_PARAMS_ADAPTER.validate_python(_litellm_params) warn_on_provider_credential_mismatch(model_name=_model_name, litellm_params=declared_params) warn_on_ignored_model_capability_flags(model_name=_model_name, litellm_params=declared_params) + warn_on_unenforced_endpoint_flag(model_name=_model_name, model_info=_model_info) deployment = without_model_capability_flags( Deployment( **deployment_info, @@ -11428,6 +11430,9 @@ class Router: declared_params: Final = deployment.litellm_params.model_dump(exclude_none=True) warn_on_provider_credential_mismatch(model_name=deployment.model_name, litellm_params=declared_params) warn_on_ignored_model_capability_flags(model_name=deployment.model_name, litellm_params=declared_params) + warn_on_unenforced_endpoint_flag( + model_name=deployment.model_name, model_info=deployment.model_info.model_dump() + ) served: Final = with_admin_api_key_secret_reference(without_model_capability_flags(deployment)) # add to model list diff --git a/litellm/router_utils/endpoint_eligibility.py b/litellm/router_utils/endpoint_eligibility.py index 018dc07e424..4a87e582c97 100644 --- a/litellm/router_utils/endpoint_eligibility.py +++ b/litellm/router_utils/endpoint_eligibility.py @@ -4,8 +4,9 @@ from contextvars import ContextVar from types import MappingProxyType from typing import Final, TypeVar -from pydantic import BaseModel, ConfigDict, ValidationError, field_validator +from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError, field_validator +from litellm._logging import verbose_router_logger from litellm.exceptions import EndpointNotSupportedByModelError CHAT_ENDPOINT_BY_CALL: Final[Mapping[str, str]] = MappingProxyType( @@ -31,12 +32,14 @@ CHAT_ENDPOINT_BY_CALL: Final[Mapping[str, str]] = MappingProxyType( def _endpoint_path(endpoint: str) -> str: - rooted: Final = "/" + endpoint.strip().lstrip("/") - return rooted.removeprefix("/openai").removeprefix("/v1beta").removeprefix("/v1") + rooted: Final = "/" + endpoint.strip().strip("/").casefold() + path: Final = rooted.removeprefix("/openai").removeprefix("/v1beta").removeprefix("/v1") + return "/" + path.rsplit(":", 1)[-1] if ":" in path else path CHAT_ENDPOINTS: Final = frozenset(_endpoint_path(endpoint) for endpoint in CHAT_ENDPOINT_BY_CALL.values()) _Deployment: Final = TypeVar("_Deployment", bound=Mapping[str, object]) +_ENFORCE_FLAG: Final = TypeAdapter(bool | None) _REQUEST_SELECTION: Final[ContextVar[tuple[str | None, str | None]]] = ContextVar( "router_request_endpoint", default=(None, None) ) @@ -65,14 +68,34 @@ def endpoint_selection(call_type: object, module: object = None, model: object = _REQUEST_SELECTION.reset(token) -def non_chat_endpoints(deployment: Mapping[str, object]) -> frozenset[str] | None: +def _enforced_endpoints(model_info: object) -> tuple[str, ...] | None: try: - declaration: Final = _EndpointDeclaration.model_validate(deployment.get("model_info")) + declaration: Final = _EndpointDeclaration.model_validate(model_info) except ValidationError: return None - if not declaration.enforce_supported_endpoints or declaration.supported_endpoints is None: + return declaration.supported_endpoints if declaration.enforce_supported_endpoints else None + + +def warn_on_unenforced_endpoint_flag(model_name: str, model_info: Mapping[str, object]) -> str | None: + try: + flagged: Final = _ENFORCE_FLAG.validate_python(model_info.get("enforce_supported_endpoints")) + except ValidationError: return None - endpoints: Final = frozenset(declaration.supported_endpoints) + if not flagged or _enforced_endpoints(model_info) is not None: + return None + warning: Final = ( + f"Deployment '{model_name}' sets enforce_supported_endpoints without a supported_endpoints list of " + 'strings, so litellm enforces nothing for it. Set supported_endpoints, e.g. ["/v1/embeddings"].' + ) + verbose_router_logger.warning(warning) + return warning + + +def non_chat_endpoints(deployment: Mapping[str, object]) -> frozenset[str] | None: + declared: Final = _enforced_endpoints(deployment.get("model_info")) + if declared is None: + return None + endpoints: Final = frozenset(declared) normalized: Final = frozenset(_endpoint_path(endpoint) for endpoint in endpoints) return endpoints if normalized.isdisjoint(CHAT_ENDPOINTS) else None diff --git a/tests/unit/router_utils/test_endpoint_eligibility.py b/tests/unit/router_utils/test_endpoint_eligibility.py index a208ca946e1..5e5d77254d0 100644 --- a/tests/unit/router_utils/test_endpoint_eligibility.py +++ b/tests/unit/router_utils/test_endpoint_eligibility.py @@ -5,7 +5,11 @@ import pytest from pydantic import TypeAdapter from litellm.exceptions import EndpointNotSupportedByModelError -from litellm.router_utils.endpoint_eligibility import eligible_deployments, endpoint_selection +from litellm.router_utils.endpoint_eligibility import ( + eligible_deployments, + endpoint_selection, + warn_on_unenforced_endpoint_flag, +) _DEPLOYMENT: Final = TypeAdapter(list[dict[str, object]]) @@ -96,6 +100,9 @@ def test_nested_non_chat_calls_do_not_inherit_the_outer_chat_restriction() -> No "chat/completions", "v1/responses", " /v1/chat/completions ", + "/v1/chat/completions/", + "/V1/Chat/Completions", + "/v1beta/models/gemini-3-flash:generateContent", ], ) def test_chat_endpoint_declarations_keep_their_meaning_across_prefixes(endpoint: str) -> None: @@ -103,3 +110,30 @@ def test_chat_endpoint_declarations_keep_their_meaning_across_prefixes(endpoint: with endpoint_selection("acompletion"): result: Final = eligible_deployments("shared", (deployment,)) assert result == (deployment,) + + +@pytest.mark.parametrize( + "info", + [ + {"enforce_supported_endpoints": True}, + {"enforce_supported_endpoints": True, "supported_endpoints": ["/v1/embeddings", None]}, + {"enforce_supported_endpoints": "true", "supported_endpoints": 7}, + ], +) +def test_a_flag_without_a_usable_endpoint_list_is_reported(info: Mapping[str, object]) -> None: + warning: Final = warn_on_unenforced_endpoint_flag("embed-only", info) + assert warning is not None + assert "'embed-only'" in warning + + +@pytest.mark.parametrize( + "info", + [ + {"enforce_supported_endpoints": True, "supported_endpoints": ["/v1/embeddings"]}, + {"enforce_supported_endpoints": True, "supported_endpoints": "/v1/embeddings"}, + {"enforce_supported_endpoints": False}, + {"supported_endpoints": ["/v1/embeddings", None]}, + ], +) +def test_an_enforceable_or_unflagged_deployment_is_not_reported(info: Mapping[str, object]) -> None: + assert warn_on_unenforced_endpoint_flag("embed-only", info) is None diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 2d8199278ea..0c94a4acf02 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -31366,3 +31366,26 @@ def test_router_keeps_supported_fallback_shapes( router: Final = Router(model_list=list(_FALLBACK_VALIDATION_MODEL_LIST), **router_kwargs) assert router.fallbacks == expected_fallbacks + + +def test_router_warns_when_a_deployment_enforces_endpoints_without_a_list(caplog): + with caplog.at_level(logging.WARNING, logger="LiteLLM Router"): + router = Router( + model_list=[ + { + "model_name": "embed-from-config", + "litellm_params": {"model": "openai/text-embedding-3-small", "api_key": "sk-fake"}, + "model_info": {"enforce_supported_endpoints": True}, + } + ] + ) + router.add_deployment( + Deployment( + model_name="embed-from-api", + litellm_params=LiteLLM_Params(model="openai/text-embedding-3-small", api_key="sk-fake"), + model_info=ModelInfo(enforce_supported_endpoints=True, supported_endpoints=["/v1/embeddings", 7]), + ) + ) + + assert "Deployment 'embed-from-config' sets enforce_supported_endpoints without" in caplog.text + assert "Deployment 'embed-from-api' sets enforce_supported_endpoints without" in caplog.text