mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(router): warn when enforce_supported_endpoints has no usable list and match chat endpoints regardless of case, trailing slash, or Gemini path
This commit is contained in:
parent
f7d35756bf
commit
b2a50dffb8
4 changed files with 93 additions and 8 deletions
|
|
@ -285,6 +285,7 @@ from litellm.router_utils.embedding_batch_size_check import (
|
|||
from litellm.router_utils.endpoint_eligibility import (
|
||||
eligible_deployments,
|
||||
endpoint_selection,
|
||||
warn_on_unenforced_endpoint_flag,
|
||||
)
|
||||
from litellm.router_utils.fallback_event_handlers import (
|
||||
MID_STREAM_FALLBACK_CONTROLS_KEY,
|
||||
|
|
@ -10625,6 +10626,7 @@ class Router:
|
|||
declared_params: Final = _LITELLM_PARAMS_ADAPTER.validate_python(_litellm_params)
|
||||
warn_on_provider_credential_mismatch(model_name=_model_name, litellm_params=declared_params)
|
||||
warn_on_ignored_model_capability_flags(model_name=_model_name, litellm_params=declared_params)
|
||||
warn_on_unenforced_endpoint_flag(model_name=_model_name, model_info=_model_info)
|
||||
deployment = without_model_capability_flags(
|
||||
Deployment(
|
||||
**deployment_info,
|
||||
|
|
@ -11428,6 +11430,9 @@ class Router:
|
|||
declared_params: Final = deployment.litellm_params.model_dump(exclude_none=True)
|
||||
warn_on_provider_credential_mismatch(model_name=deployment.model_name, litellm_params=declared_params)
|
||||
warn_on_ignored_model_capability_flags(model_name=deployment.model_name, litellm_params=declared_params)
|
||||
warn_on_unenforced_endpoint_flag(
|
||||
model_name=deployment.model_name, model_info=deployment.model_info.model_dump()
|
||||
)
|
||||
served: Final = with_admin_api_key_secret_reference(without_model_capability_flags(deployment))
|
||||
|
||||
# add to model list
|
||||
|
|
|
|||
|
|
@ -4,8 +4,9 @@ from contextvars import ContextVar
|
|||
from types import MappingProxyType
|
||||
from typing import Final, TypeVar
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, ValidationError, field_validator
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError, field_validator
|
||||
|
||||
from litellm._logging import verbose_router_logger
|
||||
from litellm.exceptions import EndpointNotSupportedByModelError
|
||||
|
||||
CHAT_ENDPOINT_BY_CALL: Final[Mapping[str, str]] = MappingProxyType(
|
||||
|
|
@ -31,12 +32,14 @@ CHAT_ENDPOINT_BY_CALL: Final[Mapping[str, str]] = MappingProxyType(
|
|||
|
||||
|
||||
def _endpoint_path(endpoint: str) -> str:
|
||||
rooted: Final = "/" + endpoint.strip().lstrip("/")
|
||||
return rooted.removeprefix("/openai").removeprefix("/v1beta").removeprefix("/v1")
|
||||
rooted: Final = "/" + endpoint.strip().strip("/").casefold()
|
||||
path: Final = rooted.removeprefix("/openai").removeprefix("/v1beta").removeprefix("/v1")
|
||||
return "/" + path.rsplit(":", 1)[-1] if ":" in path else path
|
||||
|
||||
|
||||
CHAT_ENDPOINTS: Final = frozenset(_endpoint_path(endpoint) for endpoint in CHAT_ENDPOINT_BY_CALL.values())
|
||||
_Deployment: Final = TypeVar("_Deployment", bound=Mapping[str, object])
|
||||
_ENFORCE_FLAG: Final = TypeAdapter(bool | None)
|
||||
_REQUEST_SELECTION: Final[ContextVar[tuple[str | None, str | None]]] = ContextVar(
|
||||
"router_request_endpoint", default=(None, None)
|
||||
)
|
||||
|
|
@ -65,14 +68,34 @@ def endpoint_selection(call_type: object, module: object = None, model: object =
|
|||
_REQUEST_SELECTION.reset(token)
|
||||
|
||||
|
||||
def non_chat_endpoints(deployment: Mapping[str, object]) -> frozenset[str] | None:
|
||||
def _enforced_endpoints(model_info: object) -> tuple[str, ...] | None:
|
||||
try:
|
||||
declaration: Final = _EndpointDeclaration.model_validate(deployment.get("model_info"))
|
||||
declaration: Final = _EndpointDeclaration.model_validate(model_info)
|
||||
except ValidationError:
|
||||
return None
|
||||
if not declaration.enforce_supported_endpoints or declaration.supported_endpoints is None:
|
||||
return declaration.supported_endpoints if declaration.enforce_supported_endpoints else None
|
||||
|
||||
|
||||
def warn_on_unenforced_endpoint_flag(model_name: str, model_info: Mapping[str, object]) -> str | None:
|
||||
try:
|
||||
flagged: Final = _ENFORCE_FLAG.validate_python(model_info.get("enforce_supported_endpoints"))
|
||||
except ValidationError:
|
||||
return None
|
||||
endpoints: Final = frozenset(declaration.supported_endpoints)
|
||||
if not flagged or _enforced_endpoints(model_info) is not None:
|
||||
return None
|
||||
warning: Final = (
|
||||
f"Deployment '{model_name}' sets enforce_supported_endpoints without a supported_endpoints list of "
|
||||
'strings, so litellm enforces nothing for it. Set supported_endpoints, e.g. ["/v1/embeddings"].'
|
||||
)
|
||||
verbose_router_logger.warning(warning)
|
||||
return warning
|
||||
|
||||
|
||||
def non_chat_endpoints(deployment: Mapping[str, object]) -> frozenset[str] | None:
|
||||
declared: Final = _enforced_endpoints(deployment.get("model_info"))
|
||||
if declared is None:
|
||||
return None
|
||||
endpoints: Final = frozenset(declared)
|
||||
normalized: Final = frozenset(_endpoint_path(endpoint) for endpoint in endpoints)
|
||||
return endpoints if normalized.isdisjoint(CHAT_ENDPOINTS) else None
|
||||
|
||||
|
|
|
|||
|
|
@ -5,7 +5,11 @@ import pytest
|
|||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.exceptions import EndpointNotSupportedByModelError
|
||||
from litellm.router_utils.endpoint_eligibility import eligible_deployments, endpoint_selection
|
||||
from litellm.router_utils.endpoint_eligibility import (
|
||||
eligible_deployments,
|
||||
endpoint_selection,
|
||||
warn_on_unenforced_endpoint_flag,
|
||||
)
|
||||
|
||||
_DEPLOYMENT: Final = TypeAdapter(list[dict[str, object]])
|
||||
|
||||
|
|
@ -96,6 +100,9 @@ def test_nested_non_chat_calls_do_not_inherit_the_outer_chat_restriction() -> No
|
|||
"chat/completions",
|
||||
"v1/responses",
|
||||
" /v1/chat/completions ",
|
||||
"/v1/chat/completions/",
|
||||
"/V1/Chat/Completions",
|
||||
"/v1beta/models/gemini-3-flash:generateContent",
|
||||
],
|
||||
)
|
||||
def test_chat_endpoint_declarations_keep_their_meaning_across_prefixes(endpoint: str) -> None:
|
||||
|
|
@ -103,3 +110,30 @@ def test_chat_endpoint_declarations_keep_their_meaning_across_prefixes(endpoint:
|
|||
with endpoint_selection("acompletion"):
|
||||
result: Final = eligible_deployments("shared", (deployment,))
|
||||
assert result == (deployment,)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"info",
|
||||
[
|
||||
{"enforce_supported_endpoints": True},
|
||||
{"enforce_supported_endpoints": True, "supported_endpoints": ["/v1/embeddings", None]},
|
||||
{"enforce_supported_endpoints": "true", "supported_endpoints": 7},
|
||||
],
|
||||
)
|
||||
def test_a_flag_without_a_usable_endpoint_list_is_reported(info: Mapping[str, object]) -> None:
|
||||
warning: Final = warn_on_unenforced_endpoint_flag("embed-only", info)
|
||||
assert warning is not None
|
||||
assert "'embed-only'" in warning
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"info",
|
||||
[
|
||||
{"enforce_supported_endpoints": True, "supported_endpoints": ["/v1/embeddings"]},
|
||||
{"enforce_supported_endpoints": True, "supported_endpoints": "/v1/embeddings"},
|
||||
{"enforce_supported_endpoints": False},
|
||||
{"supported_endpoints": ["/v1/embeddings", None]},
|
||||
],
|
||||
)
|
||||
def test_an_enforceable_or_unflagged_deployment_is_not_reported(info: Mapping[str, object]) -> None:
|
||||
assert warn_on_unenforced_endpoint_flag("embed-only", info) is None
|
||||
|
|
|
|||
|
|
@ -31366,3 +31366,26 @@ def test_router_keeps_supported_fallback_shapes(
|
|||
router: Final = Router(model_list=list(_FALLBACK_VALIDATION_MODEL_LIST), **router_kwargs)
|
||||
|
||||
assert router.fallbacks == expected_fallbacks
|
||||
|
||||
|
||||
def test_router_warns_when_a_deployment_enforces_endpoints_without_a_list(caplog):
|
||||
with caplog.at_level(logging.WARNING, logger="LiteLLM Router"):
|
||||
router = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "embed-from-config",
|
||||
"litellm_params": {"model": "openai/text-embedding-3-small", "api_key": "sk-fake"},
|
||||
"model_info": {"enforce_supported_endpoints": True},
|
||||
}
|
||||
]
|
||||
)
|
||||
router.add_deployment(
|
||||
Deployment(
|
||||
model_name="embed-from-api",
|
||||
litellm_params=LiteLLM_Params(model="openai/text-embedding-3-small", api_key="sk-fake"),
|
||||
model_info=ModelInfo(enforce_supported_endpoints=True, supported_endpoints=["/v1/embeddings", 7]),
|
||||
)
|
||||
)
|
||||
|
||||
assert "Deployment 'embed-from-config' sets enforce_supported_endpoints without" in caplog.text
|
||||
assert "Deployment 'embed-from-api' sets enforce_supported_endpoints without" in caplog.text
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue