scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-23 05:34:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 24

Merge 06409e2f6e into 1552b5b69f

Browse source
This commit is contained in:
Mateo Wang 2026-10-17 20:54:59 +00:00 • committed by GitHub
commit b95b8943af
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
21 changed files with 2002 additions and 28 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

29
litellm/proxy/common_request_processing.py
View file

@ -120,6 +120,7 @@ from litellm.proxy.dd_span_tagger import DDSpanTagger
from litellm.proxy.guardrails.auto_router_compression import arm_pre_call as _arm_auto_router_compression
from litellm.proxy.native_compaction import with_proxy_compaction_executor
from litellm.proxy.route_llm_request import (
endpoint_unsupported_by_model,
route_request,
)
from litellm.proxy.telemetry.request_context import bind_call_id as bind_telemetry_call_id
@ -132,6 +133,7 @@ from litellm.router import Router
from litellm.router_utils.add_retry_fallback_headers import get_hidden_params_dict
from litellm.router_utils.auto_router_model_naming import AUTO_ROUTER_MODEL_PREFIX
from litellm.router_utils.common_utils import resolve_model_group_alias, resolve_served_model
from litellm.router_utils.endpoint_eligibility import UnsupportedEndpointError
from litellm.types.guardrails import GuardrailEventHooks
from litellm.types.router import RouterRateLimitError
from litellm.types.router_weights import validate_router_weights
@ -2557,6 +2559,17 @@ class ProxyBaseLLMRequestProcessing:
original_model,
fallback_models,
)
raw_router_settings: Final = _DECODED_JSON.validate_python(
_REQUEST_FIELDS.validate_python(rate_limited_data).get("router_settings_override")
)
group_alias: Final = _DECODED_JSON.validate_python(
_REQUEST_FIELDS.validate_python(raw_router_settings).get("model_group_alias")
if isinstance(raw_router_settings, Mapping)
else None
)
key_aliases: Final[object] = cast( # cast-ok: auth aliases are dynamically typed at runtime
object, user_api_key_dict.aliases
)
try:
for fallback_model in fallback_models:
if fallback_model == original_model:
@ -2572,6 +2585,20 @@ class ProxyBaseLLMRequestProcessing:
self.data = independent_snapshot(pristine)
self.data["model"] = fallback_model
try:
probe_data = {
**_REQUEST_FIELDS.validate_python(pristine),
"model": resolve_served_model(
fallback_model,
_DECODED_JSON.validate_python(litellm.model_alias_map),
key_aliases,
group_alias,
),
}
if (
endpoint_unsupported_by_model(route_type, probe_data, llm_router, user_api_key_dict.team_id)
is not None
):
continue
return await self.common_processing_pre_call_logic(
request=request,
general_settings=general_settings,
@ -2589,6 +2616,8 @@ class ProxyBaseLLMRequestProcessing:
llm_router=llm_router,
rate_limited_model=original_model,
)
except UnsupportedEndpointError:
continue
except ProxyRateLimitError as fallback_exc:
if is_hard_per_model_limit(fallback_exc):
raise

5
litellm/proxy/common_utils/callback_utils.py
View file

@ -429,10 +429,7 @@ def get_model_group_from_litellm_kwargs(kwargs: dict) -> str | None:
def get_model_group_from_request_data(data: dict) -> str | None:
_metadata: Final = data.get("metadata", None) or {}
_model_group: Final = _metadata.get("model_group", None)
if _model_group is not None:
return _model_group
return None
return _model_group if isinstance(_model_group, str) else None
def get_remaining_tokens_and_requests_from_request_data(data: dict) -> dict[str, str]:

14
litellm/proxy/response_api_endpoints/endpoints.py
View file

@ -72,6 +72,7 @@ from litellm.proxy.pass_through_endpoints.llm_provider_handlers.batch_attributio
from litellm.proxy.route_llm_request import raise_if_required_body_param_missing
from litellm.responses.streaming_iterator import GuardedResponsesWsFrame, ResponsesWsFrameGuard
from litellm.router import Router
from litellm.router_utils.endpoint_eligibility import UnsupportedEndpointError
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.llms.openai import (
REASONING_EFFORT,
@ -1767,6 +1768,13 @@ def _routing_hints_from_first_ws_frame(first_message: str) -> Mapping[str, objec
def _responses_ws_failure_frame(failure: Exception) -> str:
if isinstance(failure, UnsupportedEndpointError):
return json.dumps(
{
"type": "error",
"error": {"type": "invalid_request_error", "message": failure.message.split("\n\n", 1)[0]},
}
)
raw_status: Final = getattr(failure, "status_code", None)
status: Final = raw_status if isinstance(raw_status, int) and not isinstance(raw_status, bool) else 500
error_type: Final = (
@ -2030,6 +2038,7 @@ async def _responses_websocket_session(
user_api_base=user_api_base,
model=resolved_model,
route_type="_aresponses_websocket",
llm_router=llm_router,
skip_guardrails=True,
)
except Exception as e:
@ -2081,7 +2090,10 @@ async def _responses_websocket_session(
original_exception=e,
request_data=routed_data,
)
await websocket.close(code=1011, reason="Internal server error")
await websocket.close(
code=1008 if isinstance(e, UnsupportedEndpointError) else 1011,
reason="Pre-call error" if isinstance(e, UnsupportedEndpointError) else "Internal server error",
)
finally:
await proxy_logging_obj.arelease_responses_websocket_slot(user_api_key_dict)

20
litellm/proxy/route_llm_request.py
View file

@ -356,6 +356,26 @@ def raise_if_required_body_param_missing(
)
def endpoint_unsupported_by_model(
route_type: str,
data: Mapping[str, object],
llm_router: LitellmRouter | None,
team_id: str | None,
) -> litellm.BadRequestError | None:
model: Final = data.get("model")
if not isinstance(model, str) or llm_router is None:
return None
metadata: Final = data.get("metadata")
request_kwargs: Final = {
**data,
"metadata": {
**(metadata if isinstance(metadata, Mapping) else {}),
"user_api_key_team_id": team_id,
},
}
return llm_router.endpoint_error_for_request(model, route_type, request_kwargs)
class MockTestingParamsDisabledError(HTTPException):
def __init__(self, params: tuple[str, ...]):
super().__init__(

8
litellm/proxy/spend_tracking/spend_tracking_utils.py
View file

@ -433,7 +433,11 @@ def _sl_attribution_fallback(
) -> str:
if standard_logging_payload is None:
return ""
return standard_logging_payload.get(field) or ""
return _string_or_empty(standard_logging_payload.get(field))
def _string_or_empty(value: object) -> str:
return value if isinstance(value, str) else ""
def _deployment_provider(deployment: DeploymentTypedDict) -> str | None:
@ -634,7 +638,7 @@ def get_logging_payload(
_model_id: Final = metadata.get("model_info", {}).get("id", "") or _sl_attribution_fallback(
standard_logging_payload, "model_id"
)
_model_group: Final = metadata.get("model_group", "") or _sl_attribution_fallback(
_model_group: Final = _string_or_empty(metadata.get("model_group")) or _sl_attribution_fallback(
standard_logging_payload, "model_group"
)
_api_base: Final = litellm_params.get("api_base", "") or _sl_attribution_fallback(

50
litellm/router.py
View file

@ -226,6 +226,11 @@ from litellm.router_utils.cooldown_handlers import (
is_caller_timeout_408,
set_cooldown_deployments,
)
from litellm.router_utils.endpoint_eligibility import (
UnsupportedEndpointError,
eligible_deployments,
endpoint_selection,
)
from litellm.router_utils.fallback_event_handlers import (
MID_STREAM_FALLBACK_CONTROLS_KEY,
AttemptedFallbackTargets,
@ -1272,7 +1277,7 @@ class Router:
self.assistants_config = assistants_config
self.search_tools = search_tools or []
self.guardrail_list = guardrail_list or []
self.deployment_names: list = [] # names of models under litellm_params. ex. azure/chatgpt-v-2
self.deployment_names: list[str] = [] # names of models under litellm_params. ex. azure/chatgpt-v-2
self.deployment_latency_map = {}
### CACHING ###
cache_type: Literal["local", "redis", "redis-semantic", "s3", "disk"] = "local" # default to an in-memory cache
@ -5529,12 +5534,13 @@ class Router:
kwargs.setdefault("metadata", {}).update({"model_group": model})
# pick the one that is available (lowest TPM/RPM)
deployment: Final = self.get_available_deployment(
model=model,
messages=messages,
specific_deployment=kwargs.pop("specific_deployment", None),
request_kwargs=kwargs,
)
with endpoint_selection("text_completion", model=model):
deployment: Final = self.get_available_deployment(
model=model,
messages=messages,
specific_deployment=kwargs.pop("specific_deployment", None),
request_kwargs=kwargs,
)
data: Final = deployment["litellm_params"].copy()
for k, v in self.default_litellm_params.items():
@ -8689,9 +8695,18 @@ class Router:
Handler for making a call to the .completion()/.embeddings()/etc. functions.
"""
model_group: Final = kwargs.get("model")
response = original_function(*args, **kwargs)
if coroutine_checker.is_async_callable(response) or inspect.isawaitable(response):
response = await response
generic_function: Final = _OPAQUE_VALUE_ADAPTER.validate_python(kwargs.get("original_generic_function"))
endpoint_function: Final = (
generic_function if callable(generic_function) else _OPAQUE_VALUE_ADAPTER.validate_python(original_function)
)
with endpoint_selection(
getattr(endpoint_function, "__name__", None),
getattr(endpoint_function, "__module__", None),
model=_OPAQUE_VALUE_ADAPTER.validate_python(model_group),
):
response = original_function(*args, **kwargs)
if coroutine_checker.is_async_callable(response) or inspect.isawaitable(response):
response = await response
await self.increment_deployment_usage_for_response(response=response, request_kwargs=kwargs)
## PROCESS RESPONSE HEADERS
response = await self.set_response_headers(response=response, model_group=model_group, request_kwargs=kwargs)
@ -13892,6 +13907,16 @@ class Router:
deployment_model: Final = litellm_params.get("model")
return isinstance(deployment_model, str) and classify_strategy_router_model(deployment_model) is not None
def endpoint_error_for_request(
self, model: str, call_type: str, request_kwargs: Mapping[str, object]
) -> litellm.BadRequestError | None:
try:
with endpoint_selection(call_type, model=model):
self._common_checks_available_deployment(model=model, request_kwargs=request_kwargs)
except litellm.BadRequestError as error:
return error
return None
def _common_checks_available_deployment(
self,
model: str,
@ -14076,7 +14101,10 @@ class Router:
model=model,
llm_provider="",
)
return selectable
eligible: Final = eligible_deployments(model, selectable)
if isinstance(eligible, UnsupportedEndpointError):
raise eligible
return list(eligible)
def _filter_reserved_deployments(
self,

102
litellm/router_utils/endpoint_eligibility.py Normal file
View file

@ -0,0 +1,102 @@
from collections.abc import Generator, Mapping, Sequence
from contextlib import contextmanager
from contextvars import ContextVar
from types import MappingProxyType
from typing import Final, TypeVar
from pydantic import BaseModel, ConfigDict, ValidationError
from litellm.exceptions import BadRequestError
CHAT_ENDPOINT_BY_CALL: Final[Mapping[str, str]] = MappingProxyType(
{
"completion": "/v1/chat/completions",
"acompletion": "/v1/chat/completions",
"_completion": "/v1/chat/completions",
"_acompletion": "/v1/chat/completions",
"text_completion": "/v1/completions",
"atext_completion": "/v1/completions",
"_atext_completion": "/v1/completions",
"anthropic_messages": "/v1/messages",
"aresponses": "/v1/responses",
"agenerate_content": "generateContent",
"agenerate_content_stream": "streamGenerateContent",
"acreate_interaction": "/v1beta/interactions",
"litellm.interactions.main.acreate": "/v1beta/interactions",
"litellm.interactions.main.create": "/v1beta/interactions",
"_aresponses_websocket": "/v1/responses",
"acompact_responses": "/v1/responses/compact",
}
)
def _endpoint_path(endpoint: str) -> str:
return endpoint.removeprefix("/openai").removeprefix("/v1beta").removeprefix("/v1")
CHAT_ENDPOINTS: Final = frozenset(_endpoint_path(endpoint) for endpoint in CHAT_ENDPOINT_BY_CALL.values())
_Deployment: Final = TypeVar("_Deployment", bound=Mapping[str, object])
_REQUEST_SELECTION: Final[ContextVar[tuple[str | None, str | None]]] = ContextVar(
"router_request_endpoint", default=(None, None)
)
class _EndpointDeclaration(BaseModel):
model_config = ConfigDict(frozen=True)
enforce_supported_endpoints: bool = False
supported_endpoints: tuple[str, ...] | None = None
class UnsupportedEndpointError(BadRequestError):
def __init__(self, message: str, model: str, llm_provider: str = "") -> None:
super().__init__( # pyright: ignore[reportUnknownMemberType] # inherited public exception constructor is untyped
message=message, model=model, llm_provider=llm_provider
)
self.message = message
self.type = "invalid_request_error"
self.param = "model"
self.code = "400"
@contextmanager
def endpoint_selection(call_type: object, module: object = None, model: object = None) -> Generator[None]:
qualified_call: Final = f"{module}.{call_type}" if module == "litellm.interactions.main" else call_type
endpoint: Final = CHAT_ENDPOINT_BY_CALL.get(qualified_call) if isinstance(qualified_call, str) else None
token: Final = _REQUEST_SELECTION.set((endpoint, model if isinstance(model, str) else None))
try:
yield
finally:
_REQUEST_SELECTION.reset(token)
def non_chat_endpoints(deployment: Mapping[str, object]) -> frozenset[str] | None:
try:
declaration: Final = _EndpointDeclaration.model_validate(deployment.get("model_info"))
except ValidationError:
return None
if not declaration.enforce_supported_endpoints or declaration.supported_endpoints is None:
return None
endpoints: Final = frozenset(declaration.supported_endpoints)
normalized: Final = frozenset(_endpoint_path(endpoint) for endpoint in endpoints)
return endpoints if normalized.isdisjoint(CHAT_ENDPOINTS) else None
def eligible_deployments(
model: str, deployments: Sequence[_Deployment]
) -> tuple[_Deployment, ...] | UnsupportedEndpointError:
endpoint, requested_model = _REQUEST_SELECTION.get()
if endpoint is None or not deployments:
return tuple(deployments)
eligible: Final = tuple(deployment for deployment in deployments if non_chat_endpoints(deployment) is None)
if eligible:
return eligible
requested: Final = requested_model or model
supported: Final = ", ".join(
sorted(frozenset[str]().union(*(non_chat_endpoints(deployment) or frozenset() for deployment in deployments)))
)
return UnsupportedEndpointError(
message=f"{endpoint} is not supported by model={requested}. Supported endpoints: {supported or 'none'}",
model=model,
llm_provider="",
)

1
litellm/types/router.py
View file

@ -255,6 +255,7 @@ def _whole_float_as_int(value: object) -> object:
class ModelInfo(MirroredPricingParams):
enforce_supported_endpoints: bool = False
id: str | None # Allow id to be optional on input, but it will always be present as a str in the model instance
db_model: bool = False # used for proxy - to separate models which are stored in the db vs. config.
updated_at: datetime.datetime | None = None

1
tests/e2e/coverage_registry/llm_nonconversational.yaml
View file

@ -2,6 +2,7 @@
- {id: llm.completions.openai.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: completions, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_completions_endpoint_e2e.py", rationale: "Legacy text /completions endpoint, second-highest production request volume"}
- {id: llm.embeddings.openai.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: embeddings, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_embeddings_endpoint_e2e.py:23", rationale: "Core endpoint, live vector response"}
- {id: llm.embeddings.openai.input_validation.nonstream.works, module: llm, tier: P1, subject_endpoint: embeddings, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], source: "vendor strategy §9.3 / LIT-4778", rationale: "Missing model/input on /embeddings return client errors"}
- {id: llm.embeddings.openai.supported_endpoints_gate.nonstream.works, module: llm, tier: P1, subject_endpoint: embeddings, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], fail_before_fix: proven, source: "test_embedding_only_supported_endpoints_e2e.py", rationale: "Explicit endpoint restrictions preserve real embeddings and route chat through a live fallback; rejection and zero-provider-call contracts live in integration/routing/test_non_chat_model_endpoint_gate.py"}
- {id: llm.embeddings.openai.basic.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: embeddings, route: openai, capability: basic, streaming: nonstream, assertions: [cost_logged], source: "SPEND_TRACKING_COVERAGE_MATRIX.md:34", rationale: "Cost tracking on embeddings"}
- {id: llm.embeddings.openai.passthrough.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: embeddings, route: openai, capability: basic, streaming: nonstream, assertions: [cost_logged], source: "test_passthrough_e2e.py", rationale: "POST /openai_passthrough/v1/embeddings is costed; the route wrote no spend row at all, so budgets never saw traffic OpenAI was billing for (GitHub issue #36646)"}
- {id: llm.embeddings.azure_openai.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: embeddings, route: azure_openai, capability: basic, streaming: nonstream, assertions: [works], source: "llms/azure/azure.py", rationale: "Azure embeddings via translation"}

104
tests/e2e/llm_translation/test_embedding_only_supported_endpoints_e2e.py Normal file
View file

@ -0,0 +1,104 @@
"""Live provider checks for embeddings and chat fallback from an explicitly restricted model."""
from __future__ import annotations
import dataclasses
from typing import Final
import pytest
from e2e_config import unique_marker
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
from lifecycle import ResourceManager
from models import LiteLLMParamsBody, ModelInfoBody, ModelMode, ModelNewBody
from openai import OpenAI
from proxy_client import ProxyClient
from sdk_clients import NO_PROXY_CACHE, SdkClients
pytestmark = pytest.mark.e2e
EMBEDDING_BACKEND: Final = "openai/text-embedding-3-small"
CHAT_FALLBACK_BACKEND: Final = "openai/gpt-5.4-nano"
EMBEDDINGS_ONLY: Final = ("/v1/embeddings",)
PROMPT: Final = "Say this is a test!"
def _subject(route: Route) -> Subject:
return Subject(
domain=Domain.LLM_TRANSLATION,
route=route,
providers=(Provider.OPENAI,),
models=(EMBEDDING_BACKEND,),
mode=Mode.NONSTREAM,
)
def _register(
proxy: ProxyClient,
resources: ResourceManager,
prefix: str,
supported_endpoints: tuple[str, ...] | None,
backend: str = EMBEDDING_BACKEND,
mode: ModelMode = "embedding",
) -> str:
model: Final = f"{prefix}-{unique_marker()}"
model_id: Final = proxy.register_model(
ModelNewBody(
model_name=model,
litellm_params=LiteLLMParamsBody(model=backend, api_key="os.environ/OPENAI_API_KEY"),
model_info=ModelInfoBody(
enforce_supported_endpoints=supported_endpoints is not None,
mode=mode,
supported_endpoints=None if supported_endpoints is None else list(supported_endpoints),
),
)
)
resources.defer(lambda: proxy.delete_model(model_id))
return model
def _openai_at(sdk: SdkClients, api_prefix: str, key: str) -> OpenAI:
return dataclasses.replace(sdk, base_url=f"{sdk.base_url}{api_prefix}").openai(key)
class TestEmbeddingOnlySupportedEndpoints:
@pytest.mark.covers("llm.embeddings.openai.supported_endpoints_gate.nonstream.works")
@meta(_subject(Route.EMBEDDINGS))
@pytest.mark.parametrize(
("prefix", "supported_endpoints"),
(
pytest.param("e2e-embed-only", EMBEDDINGS_ONLY, id="embed-only"),
pytest.param("e2e-embed-control", None, id="control"),
),
)
def test_embeddings_return_a_vector(
self,
proxy: ProxyClient,
resources: ResourceManager,
sdk: SdkClients,
prefix: str,
supported_endpoints: tuple[str, ...] | None,
) -> None:
model: Final = _register(proxy, resources, prefix, supported_endpoints)
embeddings: Final = sdk.openai(resources.key()).embeddings.create(
model=model, input=PROMPT, extra_body=NO_PROXY_CACHE
)
assert embeddings.data, f"/embeddings returned no data for {model}: {embeddings!r}"
assert any(component != 0.0 for component in embeddings.data[0].embedding), (
f"/embeddings returned an empty or all-zero vector for {model}"
)
@pytest.mark.covers("llm.embeddings.openai.supported_endpoints_gate.nonstream.works")
@meta(dataclasses.replace(_subject(Route.CHAT_COMPLETIONS), models=(EMBEDDING_BACKEND, CHAT_FALLBACK_BACKEND)))
def test_chat_completions_with_a_fallback_are_answered_by_the_fallback(
self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients
) -> None:
model: Final = _register(proxy, resources, "e2e-embed-only", EMBEDDINGS_ONLY)
fallback: Final = _register(proxy, resources, "e2e-chat-fallback", None, CHAT_FALLBACK_BACKEND, "chat")
completion: Final = _openai_at(sdk, "/v1", resources.key()).chat.completions.create(
model=model,
messages=[{"role": "user", "content": PROMPT}],
extra_body={**NO_PROXY_CACHE, "fallbacks": [fallback]},
)
assert completion.choices and completion.choices[0].message.content, (
f"the fallback {fallback} should answer a chat call to {model}, got {completion!r}"
)

3
tests/e2e/models.py
View file

@ -1423,10 +1423,11 @@ class LiteLLMParamsBody(BaseModel):
order: int | None = None
ModelMode = Literal["chat", "batch", "realtime", "image_generation"]
ModelMode = Literal["chat", "embedding", "batch", "realtime", "image_generation"]
class ModelInfoBody(BaseModel):
enforce_supported_endpoints: bool | None = None
# id is left unset so the proxy assigns a unique model_id per deployment.
# Pinning it to the model_name made re-registrations of a fixed-name model
# (e.g. the batch suite's openai-batch) collide on the model_id unique

1479
tests/integration/routing/test_non_chat_model_endpoint_gate.py Normal file
View file

File diff suppressed because it is too large Load diff

13
tests/unit/proxy/common_utils/test_callback_utils.py
View file

@ -49,6 +49,19 @@ def test_get_remaining_tokens_and_requests_from_request_data():
}
@pytest.mark.parametrize("client_model_group", [7, ["embed-only"], {"name": "embed-only"}])
def test_remaining_limit_headers_ignore_a_model_group_that_is_not_a_string(client_model_group: object):
casedata: Final = {
"metadata": {
"model_group": client_model_group,
f"litellm-key-remaining-requests-{client_model_group}": 100,
f"litellm-key-remaining-tokens-{client_model_group}": 200,
}
}
assert get_remaining_tokens_and_requests_from_request_data(casedata) == {}
@patch(
"litellm.proxy.common_utils.callback_utils.CustomLogger.get_callback_env_vars",
return_value=["API_KEY", "MISSING_VAR"],

2
tests/unit/proxy/response_api_endpoints/test_endpoints.py
View file

@ -2479,6 +2479,8 @@ class TestResponsesWSFirstFrameModelAuth:
)
class TestReadWSModelFromFirstFrameErrors:
@pytest.mark.asyncio
async def test_transport_error_first_frame_closes_with_internal_error(self):

22
tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py
View file

@ -5056,6 +5056,28 @@ def test_get_logging_payload_router_rejected_request_takes_provider_from_model_g
assert payload["custom_llm_provider"] == expected_provider
@pytest.mark.parametrize("client_model_group", [7, ["embed-only"], {"name": "embed-only"}])
def test_get_logging_payload_drops_a_client_model_group_that_is_not_a_string(client_model_group: object):
payload: Final = get_logging_payload(
kwargs={
"model": "embed-only",
"litellm_params": {
"metadata": {"user_api_key": "test-key", "model_group": client_model_group, "status": "failure"}
},
"standard_logging_object": {
**_make_failed_request_standard_logging_payload(),
"model_group": client_model_group,
},
},
response_obj={},
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
llm_router=None,
)
assert payload["model_group"] == ""
def test_get_logging_payload_router_rejected_request_without_router_leaves_provider_empty():
assert _router_rejected_failure_payload("openai-group", None)["custom_llm_provider"] == ""

19
tests/unit/proxy/test_common_request_processing.py
View file

@ -452,7 +452,7 @@ class TestProxyBaseLLMRequestProcessing:
mock_add_litellm_data_to_request,
)
mock_general_settings = {}
mock_user_api_key_dict = MagicMock(spec=UserAPIKeyAuth)
mock_user_api_key_dict = ProxyUserAPIKeyAuth()
mock_proxy_config = MagicMock(spec=ProxyConfig)
route_type = "acompletion"
@ -574,7 +574,7 @@ class TestProxyBaseLLMRequestProcessing:
returned_data, logging_obj = await processing_obj.common_processing_pre_call_logic(
request=mock_request,
general_settings={},
user_api_key_dict=MagicMock(spec=UserAPIKeyAuth),
user_api_key_dict=ProxyUserAPIKeyAuth(),
proxy_logging_obj=mock_proxy_logging_obj,
proxy_config=MagicMock(spec=ProxyConfig),
route_type=route_type,
@ -713,6 +713,9 @@ class TestProxyBaseLLMRequestProcessing:
assert returned_data["metadata"]["tags"] == ["existing-tag"]
tag_budget_check.assert_not_awaited()
@pytest.mark.asyncio
async def test_common_processing_pre_call_logic_skips_guardrail_tag_budget_check_for_zero_cost_model(
self, monkeypatch
@ -1020,7 +1023,7 @@ class TestProxyBaseLLMRequestProcessing:
)
mock_general_settings = {}
mock_user_api_key_dict = MagicMock(spec=UserAPIKeyAuth)
mock_user_api_key_dict = ProxyUserAPIKeyAuth()
mock_proxy_config = MagicMock(spec=ProxyConfig)
mock_router_settings = {
@ -2042,7 +2045,7 @@ class TestProxyBaseLLMRequestProcessing:
mock_add_litellm_data_to_request,
)
mock_general_settings = {}
mock_user_api_key_dict = MagicMock(spec=UserAPIKeyAuth)
mock_user_api_key_dict = ProxyUserAPIKeyAuth()
mock_proxy_config = MagicMock(spec=ProxyConfig)
route_type = "acompletion"
@ -7423,6 +7426,7 @@ class TestPreCallWithFallbacksOnLocalRateLimit:
return processor.data, logging_obj
mock_router = MagicMock()
mock_router.endpoint_error_for_request.return_value = None
mock_router.fallbacks = [{"gpt-4": ["gpt-3.5-turbo"]}]
with patch.object(
@ -7464,6 +7468,7 @@ class TestPreCallWithFallbacksOnLocalRateLimit:
)
mock_router = MagicMock()
mock_router.endpoint_error_for_request.return_value = None
mock_router.fallbacks = None
with patch.object(
@ -7503,6 +7508,7 @@ class TestPreCallWithFallbacksOnLocalRateLimit:
)
mock_router = MagicMock()
mock_router.endpoint_error_for_request.return_value = None
mock_router.fallbacks = [{"gpt-4": ["gpt-3.5-turbo", "claude-3-haiku"]}]
with patch.object(
@ -7530,6 +7536,7 @@ class TestPreCallWithFallbacksOnLocalRateLimit:
assert processor.data["model"] == "gpt-4"
@pytest.mark.asyncio
async def test_fallback_uses_key_level_router_settings(self):
from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
@ -7546,6 +7553,7 @@ class TestPreCallWithFallbacksOnLocalRateLimit:
return processor.data, MagicMock()
mock_router = MagicMock()
mock_router.endpoint_error_for_request.return_value = None
mock_router.fallbacks = [{"gpt-4": ["gpt-3.5-turbo"]}]
user_api_key_dict = MagicMock()
@ -7593,6 +7601,7 @@ class TestPreCallWithFallbacksOnLocalRateLimit:
)
mock_router = MagicMock()
mock_router.endpoint_error_for_request.return_value = None
mock_router.fallbacks = [{"gpt-4": ["gpt-3.5-turbo"]}]
with patch.object(
@ -7637,6 +7646,7 @@ class TestPreCallWithFallbacksOnLocalRateLimit:
raise ValueError("unexpected auth failure on fallback")
mock_router = MagicMock()
mock_router.endpoint_error_for_request.return_value = None
mock_router.fallbacks = [{"gpt-4": ["gpt-3.5-turbo"]}]
with patch.object(
@ -7746,6 +7756,7 @@ class TestPreCallWithFallbacksOnLocalRateLimit:
return processor.data, MagicMock()
mock_router = MagicMock()
mock_router.endpoint_error_for_request.return_value = None
mock_router.fallbacks = [{primary_model: [fallback_model]}]
with patch.object(

9
tests/unit/proxy/test_route_llm_request.py
View file

@ -10,7 +10,10 @@ from unittest.mock import MagicMock
from fastapi import HTTPException
from litellm.proxy.route_llm_request import ProxyModelNotFoundError, route_request
from litellm.proxy.route_llm_request import (
ProxyModelNotFoundError,
route_request,
)
@pytest.mark.parametrize(
@ -1524,6 +1527,8 @@ async def test_route_request_read_through_recovers_model_created_on_sibling_repl
assert table.find_many_wheres[0] == {"model_name": model_name}
@pytest.mark.asyncio
async def test_route_request_unknown_model_raises_and_hits_db_once_within_ttl(monkeypatch):
import litellm
@ -1765,6 +1770,8 @@ async def test_route_request_router_settings_override_skips_null_fields():
assert "model_group_retry_policy" not in call_kwargs
def _router_with_defaults(default_litellm_params: dict[str, object]):
import litellm

87
tests/unit/router_utils/test_endpoint_eligibility.py Normal file
View file

@ -0,0 +1,87 @@
from collections.abc import Mapping
from typing import Final
import pytest
from pydantic import TypeAdapter
from litellm.router_utils.endpoint_eligibility import UnsupportedEndpointError, eligible_deployments, endpoint_selection
_DEPLOYMENT: Final = TypeAdapter(list[dict[str, object]])
def _deployment(info: Mapping[str, object], identity: str = "embedding") -> dict[str, object]:
return _DEPLOYMENT.validate_python(
[
{
"model_name": "shared",
"litellm_params": {"model": "openai/test-model"},
"model_info": {"id": identity, **info},
}
]
)[0]
@pytest.mark.parametrize("call_type", ["acompletion", "anthropic_messages", "aresponses", "acreate_interaction"])
def test_explicit_restriction_excludes_an_embedding_deployment_from_a_mixed_group(call_type: str) -> None:
embedding: Final = _deployment({"enforce_supported_endpoints": True, "supported_endpoints": ["/v1/embeddings"]})
chat: Final = _deployment({"supported_endpoints": ["/v1/chat/completions"]}, "chat")
with endpoint_selection(call_type):
result: Final = eligible_deployments("shared", (embedding, chat))
assert result == (chat,)
@pytest.mark.parametrize(
"info",
[
{"mode": "audio_speech", "supported_endpoints": ["/v1/audio/speech"]},
{"enforce_supported_endpoints": False, "supported_endpoints": ["/v1/embeddings"]},
{"enforce_supported_endpoints": True},
{"enforce_supported_endpoints": True, "supported_endpoints": None},
{"enforce_supported_endpoints": True, "supported_endpoints": "invalid"},
{"enforce_supported_endpoints": True, "supported_endpoints": ["/v1/messages"]},
],
)
def test_advisory_or_incomplete_metadata_preserves_routing(info: Mapping[str, object]) -> None:
deployment: Final = _deployment(info)
with endpoint_selection("acompletion"):
result: Final = eligible_deployments("shared", (deployment,))
assert result == (deployment,)
def test_unsupported_error_explains_the_requested_operation_and_available_endpoints() -> None:
deployment: Final = _deployment({"enforce_supported_endpoints": True, "supported_endpoints": ["/v1/embeddings"]})
with endpoint_selection("acreate_interaction"):
result: Final = eligible_deployments("shared", (deployment,))
assert isinstance(result, UnsupportedEndpointError)
assert result.status_code == 400
assert "/v1beta/interactions" in result.message
assert "/v1/embeddings" in result.message
@pytest.mark.parametrize("call_type", ["aembedding", "aspeech", "_arealtime", "aget_responses"])
def test_chat_restriction_does_not_block_other_operations(call_type: str) -> None:
deployment: Final = _deployment({"enforce_supported_endpoints": True, "supported_endpoints": ["/v1/embeddings"]})
with endpoint_selection(call_type):
result: Final = eligible_deployments("shared", (deployment,))
assert result == (deployment,)
def test_nested_non_chat_calls_do_not_inherit_the_outer_chat_restriction() -> None:
deployment: Final = _deployment({"enforce_supported_endpoints": True, "supported_endpoints": ["/v1/embeddings"]})
with endpoint_selection("acompletion"):
with endpoint_selection("aembedding"):
embedding_result: Final = eligible_deployments("shared", (deployment,))
chat_result: Final = eligible_deployments("shared", (deployment,))
assert embedding_result == (deployment,)
assert isinstance(chat_result, UnsupportedEndpointError)
@pytest.mark.parametrize(
"endpoint",
["/messages", "/interactions", "/responses/compact", "/openai/v1/responses/compact", "/openai/v1/chat/completions"],
)
def test_chat_endpoint_declarations_keep_their_meaning_across_prefixes(endpoint: str) -> None:
deployment: Final = _deployment({"enforce_supported_endpoints": True, "supported_endpoints": [endpoint]})
with endpoint_selection("acompletion"):
result: Final = eligible_deployments("shared", (deployment,))
assert result == (deployment,)

51
tests/unit/test_router/test_router.py
View file

@ -11285,6 +11285,57 @@ def test_is_deployment_blocked_static_helper_reflects_blocked_flag():
)
def test_synchronous_text_completion_rejects_a_restricted_model_before_the_provider() -> None:
model: Final = "text-completion-openai/unit-text-model"
router: Final = Router(
model_list=[
{
"model_name": model,
"litellm_params": {
"model": model,
"api_base": "https://sync-router.invalid/v1",
"api_key": "unit-provider-key",
},
"model_info": {"enforce_supported_endpoints": True, "supported_endpoints": ["/v1/embeddings"]},
}
],
num_retries=0,
)
with respx.mock(assert_all_called=False) as upstream:
provider: Final = upstream.post("https://sync-router.invalid/v1/completions").respond(
200,
json={
"id": "cmpl-unit",
"object": "text_completion",
"created": 1,
"model": "unit-text-model",
"choices": [{"text": "provider called", "index": 0, "finish_reason": "stop", "logprobs": None}],
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
},
)
with pytest.raises(litellm.BadRequestError, match="/v1/completions is not supported"):
router.text_completion(model=model, prompt="test")
assert provider.call_count == 0
def test_endpoint_preflight_returns_unavailable_model_errors_as_values() -> None:
router: Final = Router(
model_list=[
{
"model_name": "available",
"litellm_params": {"model": "openai/gpt-5.4-nano", "api_key": "unit-provider-key"},
}
],
num_retries=0,
)
missing: Final = "unregistered-fallback"
error: Final = router.endpoint_error_for_request(missing, "acompletion", {})
assert isinstance(error, litellm.BadRequestError)
assert error.status_code == 400
assert error.model == missing
assert router.endpoint_error_for_request("available", "acompletion", {}) is None
def test_routing_groups_legacy_property_forwards_to_public_attribute():
router = _router_with_two_deployments([False, False])

6
tests/unit/test_router_model_cost_isolation.py
View file

@ -2211,10 +2211,8 @@ def test_a_reload_rebuilds_exactly_what_a_fresh_boot_registered() -> None:
f"the rebuild changed or dropped a field the boot registration wrote: "
f"{ {k: (v, rebuilt.get(k)) for k, v in at_boot.items() if rebuilt.get(k) != v} }"
)
assert {field: rebuilt[field] for field in set(rebuilt) - set(at_boot)} == {
"db_model": False,
"member_auto_router": False,
}
replay_defaults: Final = {field: rebuilt[field] for field in set(rebuilt) - set(at_boot)}
assert replay_defaults.items() <= ModelInfo().model_dump(exclude_none=True).items()
assert "member_auto_router" not in litellm.model_cost["gpt-4o"]
assert router.model_list
finally:

5
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -49972,6 +49972,11 @@ export interface components {
discoverable?: boolean | null;
/** Enable Tag Filtering */
enable_tag_filtering?: boolean | null;
/**
* Enforce Supported Endpoints
* @default false
*/
enforce_supported_endpoints: boolean;
/** Id */
id: string | null;
/** Input Cost Per Character */