This commit is contained in:
Mateo Wang 2026-10-29 06:09:14 +00:00 • committed by GitHub
commit ed9f34bfb8
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
21 changed files with 2123 additions and 26 deletions

View file

@ -321,6 +321,17 @@ class UnsupportedEndpointError(BadRequestError):
pass
class EndpointNotSupportedByModelError(BadRequestError):
def __init__(self, message: str, model: str, llm_provider: str = "") -> None:
super().__init__( # pyright: ignore[reportUnknownMemberType] # inherited public exception constructor is untyped
message=message, model=model, llm_provider=llm_provider
)
self.message = message
self.type = "invalid_request_error"
self.param = "model"
self.code = "400"
class UnprocessableEntityError(openai.UnprocessableEntityError):
def __init__(
self,

View file

@ -129,6 +129,7 @@ from litellm.proxy.guardrails.auto_router_compression import arm_pre_call as _ar
from litellm.proxy.hooks.parallel_request_limiter_v3 import get_or_create_request_stash
from litellm.proxy.native_compaction import with_proxy_compaction_executor
from litellm.proxy.route_llm_request import (
endpoint_unsupported_by_model,
route_request,
)
from litellm.proxy.telemetry.request_context import bind_call_id as bind_telemetry_call_id
@ -2676,6 +2677,17 @@ class ProxyBaseLLMRequestProcessing:
original_model,
fallback_models,
)
raw_router_settings: Final = _DECODED_JSON.validate_python(
_REQUEST_FIELDS.validate_python(rate_limited_data).get("router_settings_override")
)
group_alias: Final = _DECODED_JSON.validate_python(
_REQUEST_FIELDS.validate_python(raw_router_settings).get("model_group_alias")
if isinstance(raw_router_settings, Mapping)
else None
)
key_aliases: Final[object] = cast( # cast-ok: auth aliases are dynamically typed at runtime
object, user_api_key_dict.aliases
)
try:
for fallback_model in fallback_models:
if fallback_model == original_model:
@ -2691,6 +2703,17 @@ class ProxyBaseLLMRequestProcessing:
self.data = independent_snapshot(pristine)
self.data["model"] = fallback_model
try:
probe_data = {
**_REQUEST_FIELDS.validate_python(pristine),
"model": resolve_served_model(
fallback_model,
_DECODED_JSON.validate_python(litellm.model_alias_map),
key_aliases,
group_alias,
),
}
if endpoint_unsupported_by_model(route_type, probe_data, llm_router, user_api_key_dict.team_id):
continue
return await self.common_processing_pre_call_logic(
request=request,
general_settings=general_settings,

View file

@ -436,10 +436,7 @@ def get_model_group_from_litellm_kwargs(kwargs: dict) -> str | None:
def get_model_group_from_request_data(data: dict) -> str | None:
_metadata: Final = data.get("metadata", None) or {}
_model_group: Final = _metadata.get("model_group", None)
if _model_group is not None:
return _model_group
return None
return _model_group if isinstance(_model_group, str) else None
def get_remaining_tokens_and_requests_from_request_data(data: dict) -> dict[str, str]:

View file

@ -34,6 +34,7 @@ from litellm._logging import verbose_proxy_logger
from litellm.anthropic_interface.exceptions.exception_mapping_utils import AnthropicExceptionMapping
from litellm.caching.dual_cache import DualCache
from litellm.constants import EMPTY_MAPPING, MANAGED_BACKGROUND_RESPONSE_METADATA_KEY, TERMINAL_RESPONSE_STATUSES
from litellm.exceptions import EndpointNotSupportedByModelError
from litellm.integrations.custom_guardrail import ModifyResponseException
from litellm.litellm_core_utils.core_helpers import as_str_mapping, get_metadata_variable_name_from_kwargs
from litellm.litellm_core_utils.internal_call_metadata import is_background_response, requests_background
@ -1991,6 +1992,13 @@ def _routing_hints_from_first_ws_frame(first_message: str) -> Mapping[str, objec
def _responses_ws_failure_frame(failure: Exception) -> str:
if isinstance(failure, EndpointNotSupportedByModelError):
return json.dumps(
{
"type": "error",
"error": {"type": "invalid_request_error", "message": failure.message.split("\n\n", 1)[0]},
}
)
proxy_exception: Final = handle_exception_on_proxy(failure)
status: Final = int(proxy_exception.code) if str(proxy_exception.code).isdigit() else 500
error_type: Final = (
@ -2288,6 +2296,7 @@ async def _responses_websocket_session(
user_api_base=user_api_base,
model=resolved_model,
route_type="_aresponses_websocket",
llm_router=llm_router,
skip_guardrails=True,
)
except Exception as e:
@ -2349,7 +2358,12 @@ async def _responses_websocket_session(
with contextlib.suppress(Exception):
await websocket.send_text(_responses_ws_failure_frame(e))
with contextlib.suppress(Exception):
await websocket.close(code=1011, reason="Internal server error")
await websocket.close(
code=1008 if isinstance(e, EndpointNotSupportedByModelError) else 1011,
reason="Pre-call error"
if isinstance(e, EndpointNotSupportedByModelError)
else "Internal server error",
)
return crash_row
return bill_socket_failure(outcome) if isinstance(outcome, Exception) else None

View file

@ -431,6 +431,26 @@ def raise_if_required_body_param_missing(
)
def endpoint_unsupported_by_model(
route_type: str,
data: Mapping[str, object],
llm_router: LitellmRouter | None,
team_id: str | None,
) -> bool:
model: Final = data.get("model")
if not isinstance(model, str) or llm_router is None:
return False
metadata: Final = data.get("metadata")
request_kwargs: Final = {
**data,
"metadata": {
**(metadata if isinstance(metadata, Mapping) else {}),
"user_api_key_team_id": team_id,
},
}
return not llm_router.get_routable_deployments(model, request_kwargs, call_type=route_type)
class MockTestingParamsDisabledError(HTTPException):
def __init__(self, params: tuple[str, ...]):
super().__init__(

View file

@ -461,7 +461,11 @@ def _sl_attribution_fallback(
) -> str:
if standard_logging_payload is None:
return ""
return standard_logging_payload.get(field) or ""
return _string_or_empty(standard_logging_payload.get(field))
def _string_or_empty(value: object) -> str:
return value if isinstance(value, str) else ""
def _deployment_provider(deployment: DeploymentTypedDict) -> str | None:
@ -679,7 +683,7 @@ def get_logging_payload(
_model_id: Final = metadata.get("model_info", {}).get("id", "") or _sl_attribution_fallback(
standard_logging_payload, "model_id"
)
_model_group: Final = metadata.get("model_group", "") or _sl_attribution_fallback(
_model_group: Final = _string_or_empty(metadata.get("model_group")) or _sl_attribution_fallback(
standard_logging_payload, "model_group"
)
_api_base: Final = litellm_params.get("api_base", "") or _sl_attribution_fallback(

View file

@ -282,6 +282,10 @@ from litellm.router_utils.embedding_batch_size_check import (
embedding_deployment_selection,
log_rejection_before_provider_call,
)
from litellm.router_utils.endpoint_eligibility import (
eligible_deployments,
endpoint_selection,
)
from litellm.router_utils.fallback_event_handlers import (
MID_STREAM_FALLBACK_CONTROLS_KEY,
AttemptedFallbackTargets,
@ -1421,7 +1425,7 @@ class Router:
self.assistants_config = assistants_config
self.search_tools = search_tools or []
self.guardrail_list = guardrail_list or []
self.deployment_names: list = [] # names of models under litellm_params. ex. azure/chatgpt-v-2
self.deployment_names: list[str] = [] # names of models under litellm_params. ex. azure/chatgpt-v-2
self.deployment_latency_map = {}
### CACHING ###
cache_type: Literal["local", "redis", "redis-semantic", "s3", "disk"] = "local" # default to an in-memory cache
@ -6023,12 +6027,13 @@ class Router:
kwargs.setdefault("metadata", {}).update({"model_group": model})
# pick the one that is available (lowest TPM/RPM)
deployment: Final = self.get_available_deployment(
model=model,
messages=messages,
specific_deployment=kwargs.pop("specific_deployment", None),
request_kwargs=kwargs,
)
with endpoint_selection("text_completion", model=model):
deployment: Final = self.get_available_deployment(
model=model,
messages=messages,
specific_deployment=kwargs.pop("specific_deployment", None),
request_kwargs=kwargs,
)
data: Final = deployment["litellm_params"].copy()
for k, v in self.default_litellm_params.items():
@ -6884,6 +6889,7 @@ class Router:
"""
model_group: Final = cast(str, initial_kwargs.get("model")) # cast-ok: kwargs always carries the model group
retry_kwargs: Final = mid_stream_retry_kwargs(initial_kwargs)
handler: Final = retry_kwargs.get("original_generic_function")
healthy_deployments, all_deployments = await self._async_get_healthy_deployments(
model=model_group,
parent_otel_span=get_parent_otel_span_from_kwargs(retry_kwargs),
@ -6914,7 +6920,10 @@ class Router:
)
carry_superseded_attempt_usage(retry_kwargs)
try:
response = await attempt_function(**retry_kwargs)
with endpoint_selection(
getattr(handler, "__name__", None), getattr(handler, "__module__", None), model=model_group
):
response = await attempt_function(**retry_kwargs)
except Exception as retry_error: # noqa: BLE001 # the fallback chain judges the failed retry
wrapped = _stream_fallback_error_for_raised(retry_error, model_group, False, e.llm_provider)
if wrapped is None:
@ -9284,9 +9293,18 @@ class Router:
Handler for making a call to the .completion()/.embeddings()/etc. functions.
"""
model_group: Final = kwargs.get("model")
response = original_function(*args, **kwargs)
if coroutine_checker.is_async_callable(response) or inspect.isawaitable(response):
response = await response
generic_function: Final = _OPAQUE_VALUE_ADAPTER.validate_python(kwargs.get("original_generic_function"))
endpoint_function: Final = (
generic_function if callable(generic_function) else _OPAQUE_VALUE_ADAPTER.validate_python(original_function)
)
with endpoint_selection(
getattr(endpoint_function, "__name__", None),
getattr(endpoint_function, "__module__", None),
model=_OPAQUE_VALUE_ADAPTER.validate_python(model_group),
):
response = original_function(*args, **kwargs)
if coroutine_checker.is_async_callable(response) or inspect.isawaitable(response):
response = await response
await self.increment_deployment_usage_for_response(response=response, request_kwargs=kwargs)
## PROCESS RESPONSE HEADERS
response = await self.set_response_headers(response=response, model_group=model_group, request_kwargs=kwargs)
@ -14739,9 +14757,15 @@ class Router:
"""
return [m for m in self.model_list if m["litellm_params"]["model"] == model]
def get_routable_deployments(self, model: str) -> tuple[DeploymentTypedDict, ...]:
def get_routable_deployments(
self,
model: str,
request_kwargs: Mapping[str, object] = MappingProxyType({}),
call_type: str | None = None,
) -> tuple[DeploymentTypedDict, ...]:
try:
_, candidates = self._common_checks_available_deployment(model=model, request_kwargs={}) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # its signature leaves input and the result untyped
with endpoint_selection(call_type, model=model):
_, candidates = self._common_checks_available_deployment(model=model, request_kwargs=request_kwargs) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # its signature leaves input and the result untyped
except (litellm.BadRequestError, ValueError):
return ()
if isinstance(candidates, dict):
@ -15017,7 +15041,10 @@ class Router:
model=model,
llm_provider="",
)
return selectable
eligible: Final = eligible_deployments(model, selectable)
if isinstance(eligible, tuple):
return list(eligible)
raise eligible
def _filter_reserved_deployments(
self,

View file

@ -0,0 +1,91 @@
from collections.abc import Generator, Mapping, Sequence
from contextlib import contextmanager
from contextvars import ContextVar
from types import MappingProxyType
from typing import Final, TypeVar
from pydantic import BaseModel, ConfigDict, ValidationError
from litellm.exceptions import EndpointNotSupportedByModelError
CHAT_ENDPOINT_BY_CALL: Final[Mapping[str, str]] = MappingProxyType(
{
"completion": "/v1/chat/completions",
"acompletion": "/v1/chat/completions",
"_completion": "/v1/chat/completions",
"_acompletion": "/v1/chat/completions",
"text_completion": "/v1/completions",
"atext_completion": "/v1/completions",
"_atext_completion": "/v1/completions",
"anthropic_messages": "/v1/messages",
"aresponses": "/v1/responses",
"agenerate_content": "generateContent",
"agenerate_content_stream": "streamGenerateContent",
"acreate_interaction": "/v1beta/interactions",
"litellm.interactions.main.acreate": "/v1beta/interactions",
"litellm.interactions.main.create": "/v1beta/interactions",
"_aresponses_websocket": "/v1/responses",
"acompact_responses": "/v1/responses/compact",
}
)
def _endpoint_path(endpoint: str) -> str:
return endpoint.removeprefix("/openai").removeprefix("/v1beta").removeprefix("/v1")
CHAT_ENDPOINTS: Final = frozenset(_endpoint_path(endpoint) for endpoint in CHAT_ENDPOINT_BY_CALL.values())
_Deployment: Final = TypeVar("_Deployment", bound=Mapping[str, object])
_REQUEST_SELECTION: Final[ContextVar[tuple[str | None, str | None]]] = ContextVar(
"router_request_endpoint", default=(None, None)
)
class _EndpointDeclaration(BaseModel):
model_config = ConfigDict(frozen=True)
enforce_supported_endpoints: bool | None = None
supported_endpoints: tuple[str, ...] | None = None
@contextmanager
def endpoint_selection(call_type: object, module: object = None, model: object = None) -> Generator[None]:
qualified_call: Final = f"{module}.{call_type}" if module == "litellm.interactions.main" else call_type
endpoint: Final = CHAT_ENDPOINT_BY_CALL.get(qualified_call) if isinstance(qualified_call, str) else None
token: Final = _REQUEST_SELECTION.set((endpoint, model if isinstance(model, str) else None))
try:
yield
finally:
_REQUEST_SELECTION.reset(token)
def non_chat_endpoints(deployment: Mapping[str, object]) -> frozenset[str] | None:
try:
declaration: Final = _EndpointDeclaration.model_validate(deployment.get("model_info"))
except ValidationError:
return None
if not declaration.enforce_supported_endpoints or declaration.supported_endpoints is None:
return None
endpoints: Final = frozenset(declaration.supported_endpoints)
normalized: Final = frozenset(_endpoint_path(endpoint) for endpoint in endpoints)
return endpoints if normalized.isdisjoint(CHAT_ENDPOINTS) else None
def eligible_deployments(
model: str, deployments: Sequence[_Deployment]
) -> tuple[_Deployment, ...] | EndpointNotSupportedByModelError:
endpoint, requested_model = _REQUEST_SELECTION.get()
if endpoint is None or not deployments:
return tuple(deployments)
eligible: Final = tuple(deployment for deployment in deployments if non_chat_endpoints(deployment) is None)
if eligible:
return eligible
requested: Final = requested_model or model
supported: Final = ", ".join(
sorted(frozenset[str]().union(*(non_chat_endpoints(deployment) or frozenset() for deployment in deployments)))
)
return EndpointNotSupportedByModelError(
message=f"{endpoint} is not supported by model={requested}. Supported endpoints: {supported or 'none'}",
model=model,
llm_provider="",
)

View file

@ -266,6 +266,7 @@ def _whole_float_as_int(value: object) -> object:
class ModelInfo(MirroredPricingParams):
enforce_supported_endpoints: bool | None = None
id: str | None # Allow id to be optional on input, but it will always be present as a str in the model instance
db_model: bool = False # used for proxy - to separate models which are stored in the db vs. config.
updated_at: datetime.datetime | None = None

View file

@ -2,6 +2,7 @@
- {id: llm.completions.openai.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: completions, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_completions_endpoint_e2e.py", rationale: "Legacy text /completions endpoint, second-highest production request volume"}
- {id: llm.embeddings.openai.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: embeddings, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_embeddings_endpoint_e2e.py:23", rationale: "Core endpoint, live vector response"}
- {id: llm.embeddings.openai.input_validation.nonstream.works, module: llm, tier: P1, subject_endpoint: embeddings, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], source: "vendor strategy §9.3 / LIT-4778", rationale: "Missing model/input on /embeddings return client errors"}
- {id: llm.embeddings.openai.supported_endpoints_gate.nonstream.works, module: llm, tier: P1, subject_endpoint: embeddings, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], fail_before_fix: proven, source: "test_embedding_only_supported_endpoints_e2e.py", rationale: "Explicit endpoint restrictions preserve real embeddings and route chat through a live fallback; rejection and zero-provider-call contracts live in integration/routing/test_non_chat_model_endpoint_gate.py"}
- {id: llm.embeddings.openai.basic.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: embeddings, route: openai, capability: basic, streaming: nonstream, assertions: [cost_logged], source: "SPEND_TRACKING_COVERAGE_MATRIX.md:34", rationale: "Cost tracking on embeddings"}
- {id: llm.embeddings.openai.passthrough.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: embeddings, route: openai, capability: basic, streaming: nonstream, assertions: [cost_logged], source: "test_passthrough_e2e.py", rationale: "POST /openai_passthrough/v1/embeddings is costed; the route wrote no spend row at all, so budgets never saw traffic OpenAI was billing for (GitHub issue #36646)"}
- {id: llm.embeddings.azure_openai.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: embeddings, route: azure_openai, capability: basic, streaming: nonstream, assertions: [works], source: "llms/azure/azure.py", rationale: "Azure embeddings via translation"}

View file

@ -0,0 +1,104 @@
"""Live provider checks for embeddings and chat fallback from an explicitly restricted model."""
from __future__ import annotations
import dataclasses
from typing import Final
import pytest
from e2e_config import unique_marker
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
from lifecycle import ResourceManager
from models import LiteLLMParamsBody, ModelInfoBody, ModelMode, ModelNewBody
from openai import OpenAI
from proxy_client import ProxyClient
from sdk_clients import NO_PROXY_CACHE, SdkClients
pytestmark = pytest.mark.e2e
EMBEDDING_BACKEND: Final = "openai/text-embedding-3-small"
CHAT_FALLBACK_BACKEND: Final = "openai/gpt-5.4-nano"
EMBEDDINGS_ONLY: Final = ("/v1/embeddings",)
PROMPT: Final = "Say this is a test!"
def _subject(route: Route) -> Subject:
return Subject(
domain=Domain.LLM_TRANSLATION,
route=route,
providers=(Provider.OPENAI,),
models=(EMBEDDING_BACKEND,),
mode=Mode.NONSTREAM,
)
def _register(
proxy: ProxyClient,
resources: ResourceManager,
prefix: str,
supported_endpoints: tuple[str, ...] | None,
backend: str = EMBEDDING_BACKEND,
mode: ModelMode = "embedding",
) -> str:
model: Final = f"{prefix}-{unique_marker()}"
model_id: Final = proxy.register_model(
ModelNewBody(
model_name=model,
litellm_params=LiteLLMParamsBody(model=backend, api_key="os.environ/OPENAI_API_KEY"),
model_info=ModelInfoBody(
enforce_supported_endpoints=supported_endpoints is not None,
mode=mode,
supported_endpoints=None if supported_endpoints is None else list(supported_endpoints),
),
)
)
resources.defer(lambda: proxy.delete_model(model_id))
return model
def _openai_at(sdk: SdkClients, api_prefix: str, key: str) -> OpenAI:
return dataclasses.replace(sdk, base_url=f"{sdk.base_url}{api_prefix}").openai(key)
class TestEmbeddingOnlySupportedEndpoints:
@pytest.mark.covers("llm.embeddings.openai.supported_endpoints_gate.nonstream.works")
@meta(_subject(Route.EMBEDDINGS))
@pytest.mark.parametrize(
("prefix", "supported_endpoints"),
(
pytest.param("e2e-embed-only", EMBEDDINGS_ONLY, id="embed-only"),
pytest.param("e2e-embed-control", None, id="control"),
),
)
def test_embeddings_return_a_vector(
self,
proxy: ProxyClient,
resources: ResourceManager,
sdk: SdkClients,
prefix: str,
supported_endpoints: tuple[str, ...] | None,
) -> None:
model: Final = _register(proxy, resources, prefix, supported_endpoints)
embeddings: Final = sdk.openai(resources.key()).embeddings.create(
model=model, input=PROMPT, extra_body=NO_PROXY_CACHE
)
assert embeddings.data, f"/embeddings returned no data for {model}: {embeddings!r}"
assert any(component != 0.0 for component in embeddings.data[0].embedding), (
f"/embeddings returned an empty or all-zero vector for {model}"
)
@pytest.mark.covers("llm.embeddings.openai.supported_endpoints_gate.nonstream.works")
@meta(dataclasses.replace(_subject(Route.CHAT_COMPLETIONS), models=(EMBEDDING_BACKEND, CHAT_FALLBACK_BACKEND)))
def test_chat_completions_with_a_fallback_are_answered_by_the_fallback(
self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients
) -> None:
model: Final = _register(proxy, resources, "e2e-embed-only", EMBEDDINGS_ONLY)
fallback: Final = _register(proxy, resources, "e2e-chat-fallback", None, CHAT_FALLBACK_BACKEND, "chat")
completion: Final = _openai_at(sdk, "/v1", resources.key()).chat.completions.create(
model=model,
messages=[{"role": "user", "content": PROMPT}],
extra_body={**NO_PROXY_CACHE, "fallbacks": [fallback]},
)
assert completion.choices and completion.choices[0].message.content, (
f"the fallback {fallback} should answer a chat call to {model}, got {completion!r}"
)

View file

@ -1416,7 +1416,7 @@ class LiteLLMParamsBody(BaseModel):
order: int | None = None
ModelMode = Literal["chat", "batch", "realtime", "image_generation"]
ModelMode = Literal["chat", "embedding", "batch", "realtime", "image_generation"]
class TokenCountModelBody(BaseModel, frozen=True):
@ -1425,6 +1425,7 @@ class TokenCountModelBody(BaseModel, frozen=True):
class ModelInfoBody(BaseModel):
enforce_supported_endpoints: bool | None = None
# id is left unset so the proxy assigns a unique model_id per deployment.
# Pinning it to the model_name made re-registrations of a fixed-name model
# (e.g. the batch suite's openai-batch) collide on the model_id unique

File diff suppressed because it is too large Load diff

View file

@ -49,6 +49,19 @@ def test_get_remaining_tokens_and_requests_from_request_data():
}
@pytest.mark.parametrize("client_model_group", [7, ["embed-only"], {"name": "embed-only"}])
def test_remaining_limit_headers_ignore_a_model_group_that_is_not_a_string(client_model_group: object):
casedata: Final = {
"metadata": {
"model_group": client_model_group,
f"litellm-key-remaining-requests-{client_model_group}": 100,
f"litellm-key-remaining-tokens-{client_model_group}": 200,
}
}
assert get_remaining_tokens_and_requests_from_request_data(casedata) == {}
@patch(
"litellm.proxy.common_utils.callback_utils.CustomLogger.get_callback_env_vars",
return_value=["API_KEY", "MISSING_VAR"],

View file

@ -3617,6 +3617,33 @@ class TestDeleteTeamModels:
assert any(e[0] == "delete_many" for e in prisma.events)
@pytest.mark.parametrize(
("model_info_patch", "expected_gate"),
[
pytest.param({"tpm": 5}, True, id="unrelated-field"),
pytest.param({"supported_endpoints": ["/v1/embeddings", "/v1/rerank"]}, True, id="endpoint-list-only"),
pytest.param({"enforce_supported_endpoints": False}, False, id="explicit-false"),
],
)
def test_model_info_patch_changes_the_endpoint_gate_only_when_it_sends_the_flag(
model_info_patch: dict[str, object], expected_gate: bool
):
from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model
result: Final = update_db_model(
db_model=Deployment(
model_name="embedding-only",
litellm_params=LiteLLM_Params(model="openai/text-embedding-3-small"),
model_info=ModelInfo(
id="embedding-only", enforce_supported_endpoints=True, supported_endpoints=["/v1/embeddings"]
),
),
updated_patch=updateDeployment.model_validate({"model_info": model_info_patch}),
)
assert json.loads(result["model_info"]).get("enforce_supported_endpoints") is expected_gate
def _build_db_model_for_blocked_test():
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo

View file

@ -1824,6 +1824,21 @@ def test_responses_ws_failure_frame_uses_the_http_status_for_the_same_failure(
assert frame["error"]["message"] == handle_exception_on_proxy(failure).message
def test_responses_ws_failure_frame_shows_only_the_refusal_for_a_model_that_refuses_the_endpoint():
from litellm.exceptions import EndpointNotSupportedByModelError
from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_failure_frame
refusal: Final = "/v1/responses is not supported by model=embed-only. Supported endpoints: /v1/embeddings"
failure: Final = EndpointNotSupportedByModelError(
message=f"{refusal}\n\nLiteLLM: model group 'embed-only' failed with the error above", model="embed-only"
)
assert json.loads(_responses_ws_failure_frame(failure)) == {
"type": "error",
"error": {"type": "invalid_request_error", "message": refusal},
}
class TestWSSessionCostTracking:
@pytest.mark.asyncio
async def test_router_budget_limiter_skips_arealtime_call_type(self):
@ -2928,6 +2943,8 @@ class TestResponsesWSFirstFrameModelAuth:
)
class TestReadWSModelFromFirstFrameErrors:
@pytest.mark.asyncio
async def test_transport_error_first_frame_closes_with_internal_error(self):

View file

@ -5384,6 +5384,28 @@ def test_get_logging_payload_router_rejected_request_takes_provider_from_model_g
assert payload["custom_llm_provider"] == expected_provider
@pytest.mark.parametrize("client_model_group", [7, ["embed-only"], {"name": "embed-only"}])
def test_get_logging_payload_drops_a_client_model_group_that_is_not_a_string(client_model_group: object):
payload: Final = get_logging_payload(
kwargs={
"model": "embed-only",
"litellm_params": {
"metadata": {"user_api_key": "test-key", "model_group": client_model_group, "status": "failure"}
},
"standard_logging_object": {
**_make_failed_request_standard_logging_payload(),
"model_group": client_model_group,
},
},
response_obj={},
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
llm_router=None,
)
assert payload["model_group"] == ""
def test_get_logging_payload_router_rejected_request_without_router_leaves_provider_empty():
assert _router_rejected_failure_payload("openai-group", None)["custom_llm_provider"] == ""

View file

@ -456,7 +456,7 @@ class TestProxyBaseLLMRequestProcessing:
mock_add_litellm_data_to_request,
)
mock_general_settings = {}
mock_user_api_key_dict = MagicMock(spec=UserAPIKeyAuth)
mock_user_api_key_dict = ProxyUserAPIKeyAuth()
mock_proxy_config = MagicMock(spec=ProxyConfig)
route_type = "acompletion"
@ -740,7 +740,7 @@ class TestProxyBaseLLMRequestProcessing:
returned_data, logging_obj = await processing_obj.common_processing_pre_call_logic(
request=mock_request,
general_settings={},
user_api_key_dict=MagicMock(spec=UserAPIKeyAuth),
user_api_key_dict=ProxyUserAPIKeyAuth(),
proxy_logging_obj=mock_proxy_logging_obj,
proxy_config=MagicMock(spec=ProxyConfig),
route_type=route_type,
@ -1186,7 +1186,7 @@ class TestProxyBaseLLMRequestProcessing:
)
mock_general_settings = {}
mock_user_api_key_dict = MagicMock(spec=UserAPIKeyAuth)
mock_user_api_key_dict = ProxyUserAPIKeyAuth()
mock_proxy_config = MagicMock(spec=ProxyConfig)
mock_router_settings = {
@ -2208,7 +2208,7 @@ class TestProxyBaseLLMRequestProcessing:
mock_add_litellm_data_to_request,
)
mock_general_settings = {}
mock_user_api_key_dict = MagicMock(spec=UserAPIKeyAuth)
mock_user_api_key_dict = ProxyUserAPIKeyAuth()
mock_proxy_config = MagicMock(spec=ProxyConfig)
route_type = "acompletion"
@ -7706,6 +7706,56 @@ class TestPreCallWithFallbacksOnLocalRateLimit:
assert processor.data["model"] == fallback_model
assert call_count == 2
@pytest.mark.asyncio
async def test_local_rate_limit_fallback_skips_an_aliased_embedding_only_model(self):
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
router: Final = Router(
model_list=[
{
"model_name": "embed-only",
"litellm_params": {"model": "openai/text-embedding-3-small", "api_key": "sk-synthetic"},
"model_info": {"enforce_supported_endpoints": True, "supported_endpoints": ["/v1/embeddings"]},
},
{"model_name": "chat-model", "litellm_params": {"model": "openai/gpt-5.4-nano", "api_key": "sk-synthetic"}},
],
fallbacks=[{"rate-limited": ["embed-alias", "chat-model"]}],
)
processor: Final = ProxyBaseLLMRequestProcessing(
data={
"model": "rate-limited",
"router_settings_override": {"model_group_alias": {"embed-alias": "embed-only"}},
}
)
attempted: Final[list[object]] = []
async def pre_call_logic(**_: object):
attempted.append(processor.data.get("model"))
if processor.data.get("model") == "rate-limited":
raise ProxyRateLimitError(detail="key rpm limit exceeded", headers={"retry-after": "30"})
return processor.data, MagicMock()
with patch.object(processor, "common_processing_pre_call_logic", side_effect=pre_call_logic):
await processor._pre_call_with_fallbacks(
request=MagicMock(),
general_settings={},
proxy_logging_obj=MagicMock(),
user_api_key_dict=ProxyUserAPIKeyAuth(),
version=None,
proxy_config=MagicMock(),
user_model=None,
user_temperature=None,
user_request_timeout=None,
user_max_tokens=None,
user_api_base=None,
model="rate-limited",
route_type="acompletion",
llm_router=router,
)
assert attempted == ["rate-limited", "chat-model"]
@pytest.mark.asyncio
async def test_raises_when_no_fallbacks_configured(self):
from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError

View file

@ -0,0 +1,88 @@
from collections.abc import Mapping
from typing import Final
import pytest
from pydantic import TypeAdapter
from litellm.exceptions import EndpointNotSupportedByModelError
from litellm.router_utils.endpoint_eligibility import eligible_deployments, endpoint_selection
_DEPLOYMENT: Final = TypeAdapter(list[dict[str, object]])
def _deployment(info: Mapping[str, object], identity: str = "embedding") -> dict[str, object]:
return _DEPLOYMENT.validate_python(
[
{
"model_name": "shared",
"litellm_params": {"model": "openai/test-model"},
"model_info": {"id": identity, **info},
}
]
)[0]
@pytest.mark.parametrize("call_type", ["acompletion", "anthropic_messages", "aresponses", "acreate_interaction"])
def test_explicit_restriction_excludes_an_embedding_deployment_from_a_mixed_group(call_type: str) -> None:
embedding: Final = _deployment({"enforce_supported_endpoints": True, "supported_endpoints": ["/v1/embeddings"]})
chat: Final = _deployment({"supported_endpoints": ["/v1/chat/completions"]}, "chat")
with endpoint_selection(call_type):
result: Final = eligible_deployments("shared", (embedding, chat))
assert result == (chat,)
@pytest.mark.parametrize(
"info",
[
{"mode": "audio_speech", "supported_endpoints": ["/v1/audio/speech"]},
{"enforce_supported_endpoints": False, "supported_endpoints": ["/v1/embeddings"]},
{"enforce_supported_endpoints": True},
{"enforce_supported_endpoints": True, "supported_endpoints": None},
{"enforce_supported_endpoints": True, "supported_endpoints": "invalid"},
{"enforce_supported_endpoints": True, "supported_endpoints": ["/v1/messages"]},
],
)
def test_advisory_or_incomplete_metadata_preserves_routing(info: Mapping[str, object]) -> None:
deployment: Final = _deployment(info)
with endpoint_selection("acompletion"):
result: Final = eligible_deployments("shared", (deployment,))
assert result == (deployment,)
def test_unsupported_error_explains_the_requested_operation_and_available_endpoints() -> None:
deployment: Final = _deployment({"enforce_supported_endpoints": True, "supported_endpoints": ["/v1/embeddings"]})
with endpoint_selection("acreate_interaction"):
result: Final = eligible_deployments("shared", (deployment,))
assert isinstance(result, EndpointNotSupportedByModelError)
assert result.status_code == 400
assert "/v1beta/interactions" in result.message
assert "/v1/embeddings" in result.message
@pytest.mark.parametrize("call_type", ["aembedding", "aspeech", "_arealtime", "aget_responses"])
def test_chat_restriction_does_not_block_other_operations(call_type: str) -> None:
deployment: Final = _deployment({"enforce_supported_endpoints": True, "supported_endpoints": ["/v1/embeddings"]})
with endpoint_selection(call_type):
result: Final = eligible_deployments("shared", (deployment,))
assert result == (deployment,)
def test_nested_non_chat_calls_do_not_inherit_the_outer_chat_restriction() -> None:
deployment: Final = _deployment({"enforce_supported_endpoints": True, "supported_endpoints": ["/v1/embeddings"]})
with endpoint_selection("acompletion"):
with endpoint_selection("aembedding"):
embedding_result: Final = eligible_deployments("shared", (deployment,))
chat_result: Final = eligible_deployments("shared", (deployment,))
assert embedding_result == (deployment,)
assert isinstance(chat_result, EndpointNotSupportedByModelError)
@pytest.mark.parametrize(
"endpoint",
["/messages", "/interactions", "/responses/compact", "/openai/v1/responses/compact", "/openai/v1/chat/completions"],
)
def test_chat_endpoint_declarations_keep_their_meaning_across_prefixes(endpoint: str) -> None:
deployment: Final = _deployment({"enforce_supported_endpoints": True, "supported_endpoints": [endpoint]})
with endpoint_selection("acompletion"):
result: Final = eligible_deployments("shared", (deployment,))
assert result == (deployment,)

View file

@ -12189,6 +12189,67 @@ def test_is_deployment_blocked_static_helper_reflects_blocked_flag():
)
def test_synchronous_text_completion_rejects_a_restricted_model_before_the_provider() -> None:
model: Final = "text-completion-openai/unit-text-model"
router: Final = Router(
model_list=[
{
"model_name": model,
"litellm_params": {
"model": model,
"api_base": "https://sync-router.invalid/v1",
"api_key": "unit-provider-key",
},
"model_info": {"enforce_supported_endpoints": True, "supported_endpoints": ["/v1/embeddings"]},
}
],
num_retries=0,
)
with respx.mock(assert_all_called=False) as upstream:
provider: Final = upstream.post("https://sync-router.invalid/v1/completions").respond(
200,
json={
"id": "cmpl-unit",
"object": "text_completion",
"created": 1,
"model": "unit-text-model",
"choices": [{"text": "provider called", "index": 0, "finish_reason": "stop", "logprobs": None}],
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
},
)
with pytest.raises(litellm.BadRequestError, match="/v1/completions is not supported"):
router.text_completion(model=model, prompt="test")
assert provider.call_count == 0
def test_routable_deployments_leave_out_models_that_refuse_the_call_type() -> None:
router: Final = Router(
model_list=[
{
"model_name": "available",
"litellm_params": {"model": "openai/gpt-5.4-nano", "api_key": "unit-provider-key"},
},
{
"model_name": "embed-only",
"litellm_params": {"model": "openai/text-embedding-3-small", "api_key": "unit-provider-key"},
"model_info": {"enforce_supported_endpoints": True, "supported_endpoints": ["/v1/embeddings"]},
},
],
num_retries=0,
)
def routable(model: str, call_type: str | None) -> tuple[str, ...]:
return tuple(
deployment["model_name"] for deployment in router.get_routable_deployments(model, {}, call_type=call_type)
)
assert routable("unregistered-fallback", "acompletion") == ()
assert routable("available", "acompletion") == ("available",)
assert routable("embed-only", "acompletion") == ()
assert routable("embed-only", "aembedding") == ("embed-only",)
assert routable("embed-only", None) == ("embed-only",)
def test_routing_groups_legacy_property_forwards_to_public_attribute():
router = _router_with_two_deployments([False, False])
@ -18648,6 +18709,35 @@ async def test_anthropic_messages_stream_dropped_before_content_is_retried_withi
assert [(attempted, budget) for _, attempted, budget in provider.calls] == [(0, 2), (1, 2)]
@pytest.mark.asyncio
async def test_anthropic_messages_same_group_retry_skips_an_embedding_only_deployment():
"""The retry of a /v1/messages stream dropped before content must not land on an enforced embedding-only
deployment of the same group, even when that deployment is the group's preferred order."""
router = Router(
model_list=[
{"model_name": "glm", "litellm_params": {"model": "anthropic/glm-chat", "api_key": "sk-test", "order": 2}},
{
"model_name": "glm",
"litellm_params": {"model": "anthropic/glm-embed", "api_key": "sk-test", "order": 1},
"model_info": {"enforce_supported_endpoints": True, "supported_endpoints": ["/v1/embeddings"]},
},
],
num_retries=1,
)
provider: Final = _AnthropicMessagesScriptedProvider(
_anthropic_messages_dropped_before_content, _anthropic_messages_retried_stream
)
async def anthropic_messages(**kwargs):
return await provider(**kwargs)
stream: Final = await _anthropic_messages_stream_through_router(router, anthropic_messages)
body: Final = [chunk async for chunk in stream]
assert body == [_anthropic_messages_message_start_chunk(), _anthropic_messages_content_chunk("pong")]
assert [model for model, _, _ in provider.calls] == ["anthropic/glm-chat", "anthropic/glm-chat"]
class _AnthropicMessagesRetryStashRecorder:
def __init__(self) -> None:
self.observed: tuple[Usage | None, ...] = ()

View file

@ -50408,6 +50408,8 @@ export interface components {
discoverable?: boolean | null;
/** Enable Tag Filtering */
enable_tag_filtering?: boolean | null;
/** Enforce Supported Endpoints */
enforce_supported_endpoints?: boolean | null;
/** Explicit Cache Read Input Token Cost */
explicit_cache_read_input_token_cost?: number | null;
/** Id */