fix(router): warn when enforce_supported_endpoints has no usable list and match chat endpoints regardless of case, trailing slash, or Gemini path

This commit is contained in:
mateo-berri 2026-10-29 06:34:28 -07:00
parent f7d35756bf
commit b2a50dffb8
4 changed files with 93 additions and 8 deletions

View file

@ -285,6 +285,7 @@ from litellm.router_utils.embedding_batch_size_check import (
from litellm.router_utils.endpoint_eligibility import (
eligible_deployments,
endpoint_selection,
warn_on_unenforced_endpoint_flag,
)
from litellm.router_utils.fallback_event_handlers import (
MID_STREAM_FALLBACK_CONTROLS_KEY,
@ -10625,6 +10626,7 @@ class Router:
declared_params: Final = _LITELLM_PARAMS_ADAPTER.validate_python(_litellm_params)
warn_on_provider_credential_mismatch(model_name=_model_name, litellm_params=declared_params)
warn_on_ignored_model_capability_flags(model_name=_model_name, litellm_params=declared_params)
warn_on_unenforced_endpoint_flag(model_name=_model_name, model_info=_model_info)
deployment = without_model_capability_flags(
Deployment(
**deployment_info,
@ -11428,6 +11430,9 @@ class Router:
declared_params: Final = deployment.litellm_params.model_dump(exclude_none=True)
warn_on_provider_credential_mismatch(model_name=deployment.model_name, litellm_params=declared_params)
warn_on_ignored_model_capability_flags(model_name=deployment.model_name, litellm_params=declared_params)
warn_on_unenforced_endpoint_flag(
model_name=deployment.model_name, model_info=deployment.model_info.model_dump()
)
served: Final = with_admin_api_key_secret_reference(without_model_capability_flags(deployment))
# add to model list

View file

@ -4,8 +4,9 @@ from contextvars import ContextVar
from types import MappingProxyType
from typing import Final, TypeVar
from pydantic import BaseModel, ConfigDict, ValidationError, field_validator
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError, field_validator
from litellm._logging import verbose_router_logger
from litellm.exceptions import EndpointNotSupportedByModelError
CHAT_ENDPOINT_BY_CALL: Final[Mapping[str, str]] = MappingProxyType(
@ -31,12 +32,14 @@ CHAT_ENDPOINT_BY_CALL: Final[Mapping[str, str]] = MappingProxyType(
def _endpoint_path(endpoint: str) -> str:
rooted: Final = "/" + endpoint.strip().lstrip("/")
return rooted.removeprefix("/openai").removeprefix("/v1beta").removeprefix("/v1")
rooted: Final = "/" + endpoint.strip().strip("/").casefold()
path: Final = rooted.removeprefix("/openai").removeprefix("/v1beta").removeprefix("/v1")
return "/" + path.rsplit(":", 1)[-1] if ":" in path else path
CHAT_ENDPOINTS: Final = frozenset(_endpoint_path(endpoint) for endpoint in CHAT_ENDPOINT_BY_CALL.values())
_Deployment: Final = TypeVar("_Deployment", bound=Mapping[str, object])
_ENFORCE_FLAG: Final = TypeAdapter(bool | None)
_REQUEST_SELECTION: Final[ContextVar[tuple[str | None, str | None]]] = ContextVar(
"router_request_endpoint", default=(None, None)
)
@ -65,14 +68,34 @@ def endpoint_selection(call_type: object, module: object = None, model: object =
_REQUEST_SELECTION.reset(token)
def non_chat_endpoints(deployment: Mapping[str, object]) -> frozenset[str] | None:
def _enforced_endpoints(model_info: object) -> tuple[str, ...] | None:
try:
declaration: Final = _EndpointDeclaration.model_validate(deployment.get("model_info"))
declaration: Final = _EndpointDeclaration.model_validate(model_info)
except ValidationError:
return None
if not declaration.enforce_supported_endpoints or declaration.supported_endpoints is None:
return declaration.supported_endpoints if declaration.enforce_supported_endpoints else None
def warn_on_unenforced_endpoint_flag(model_name: str, model_info: Mapping[str, object]) -> str | None:
try:
flagged: Final = _ENFORCE_FLAG.validate_python(model_info.get("enforce_supported_endpoints"))
except ValidationError:
return None
endpoints: Final = frozenset(declaration.supported_endpoints)
if not flagged or _enforced_endpoints(model_info) is not None:
return None
warning: Final = (
f"Deployment '{model_name}' sets enforce_supported_endpoints without a supported_endpoints list of "
'strings, so litellm enforces nothing for it. Set supported_endpoints, e.g. ["/v1/embeddings"].'
)
verbose_router_logger.warning(warning)
return warning
def non_chat_endpoints(deployment: Mapping[str, object]) -> frozenset[str] | None:
declared: Final = _enforced_endpoints(deployment.get("model_info"))
if declared is None:
return None
endpoints: Final = frozenset(declared)
normalized: Final = frozenset(_endpoint_path(endpoint) for endpoint in endpoints)
return endpoints if normalized.isdisjoint(CHAT_ENDPOINTS) else None

View file

@ -5,7 +5,11 @@ import pytest
from pydantic import TypeAdapter
from litellm.exceptions import EndpointNotSupportedByModelError
from litellm.router_utils.endpoint_eligibility import eligible_deployments, endpoint_selection
from litellm.router_utils.endpoint_eligibility import (
eligible_deployments,
endpoint_selection,
warn_on_unenforced_endpoint_flag,
)
_DEPLOYMENT: Final = TypeAdapter(list[dict[str, object]])
@ -96,6 +100,9 @@ def test_nested_non_chat_calls_do_not_inherit_the_outer_chat_restriction() -> No
"chat/completions",
"v1/responses",
" /v1/chat/completions ",
"/v1/chat/completions/",
"/V1/Chat/Completions",
"/v1beta/models/gemini-3-flash:generateContent",
],
)
def test_chat_endpoint_declarations_keep_their_meaning_across_prefixes(endpoint: str) -> None:
@ -103,3 +110,30 @@ def test_chat_endpoint_declarations_keep_their_meaning_across_prefixes(endpoint:
with endpoint_selection("acompletion"):
result: Final = eligible_deployments("shared", (deployment,))
assert result == (deployment,)
@pytest.mark.parametrize(
"info",
[
{"enforce_supported_endpoints": True},
{"enforce_supported_endpoints": True, "supported_endpoints": ["/v1/embeddings", None]},
{"enforce_supported_endpoints": "true", "supported_endpoints": 7},
],
)
def test_a_flag_without_a_usable_endpoint_list_is_reported(info: Mapping[str, object]) -> None:
warning: Final = warn_on_unenforced_endpoint_flag("embed-only", info)
assert warning is not None
assert "'embed-only'" in warning
@pytest.mark.parametrize(
"info",
[
{"enforce_supported_endpoints": True, "supported_endpoints": ["/v1/embeddings"]},
{"enforce_supported_endpoints": True, "supported_endpoints": "/v1/embeddings"},
{"enforce_supported_endpoints": False},
{"supported_endpoints": ["/v1/embeddings", None]},
],
)
def test_an_enforceable_or_unflagged_deployment_is_not_reported(info: Mapping[str, object]) -> None:
assert warn_on_unenforced_endpoint_flag("embed-only", info) is None

View file

@ -31366,3 +31366,26 @@ def test_router_keeps_supported_fallback_shapes(
router: Final = Router(model_list=list(_FALLBACK_VALIDATION_MODEL_LIST), **router_kwargs)
assert router.fallbacks == expected_fallbacks
def test_router_warns_when_a_deployment_enforces_endpoints_without_a_list(caplog):
with caplog.at_level(logging.WARNING, logger="LiteLLM Router"):
router = Router(
model_list=[
{
"model_name": "embed-from-config",
"litellm_params": {"model": "openai/text-embedding-3-small", "api_key": "sk-fake"},
"model_info": {"enforce_supported_endpoints": True},
}
]
)
router.add_deployment(
Deployment(
model_name="embed-from-api",
litellm_params=LiteLLM_Params(model="openai/text-embedding-3-small", api_key="sk-fake"),
model_info=ModelInfo(enforce_supported_endpoints=True, supported_endpoints=["/v1/embeddings", 7]),
)
)
assert "Deployment 'embed-from-config' sets enforce_supported_endpoints without" in caplog.text
assert "Deployment 'embed-from-api' sets enforce_supported_endpoints without" in caplog.text