mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-29 06:33:28 +00:00
feat(proxy): reject chat routes on models declared non-chat via model_info.supported_endpoints
A model whose every deployment sets a non-chat model_info.mode and a supported_endpoints list with no chat endpoint now answers /v1/chat/completions, /v1/messages and /v1/responses with a 400 before any provider call. Models without both fields keep today's behavior.
This commit is contained in:
parent
3e16bf9518
commit
fd150ad599
7 changed files with 473 additions and 3 deletions
|
|
@ -119,6 +119,7 @@ from litellm.proxy.dd_span_tagger import DDSpanTagger
|
|||
from litellm.proxy.guardrails.auto_router_compression import arm_pre_call as _arm_auto_router_compression
|
||||
from litellm.proxy.native_compaction import with_proxy_compaction_executor
|
||||
from litellm.proxy.route_llm_request import (
|
||||
endpoint_unsupported_by_model,
|
||||
route_request,
|
||||
)
|
||||
from litellm.proxy.utils import ( # noqa: F401 # legacy module exports
|
||||
|
|
@ -2289,6 +2290,15 @@ class ProxyBaseLLMRequestProcessing:
|
|||
|
||||
self.data["litellm_logging_obj"] = logging_obj
|
||||
|
||||
unsupported_endpoint: Final = endpoint_unsupported_by_model(
|
||||
route_type=route_type,
|
||||
model=self.data.get("model"),
|
||||
llm_router=llm_router,
|
||||
team_id=user_api_key_dict.team_id,
|
||||
)
|
||||
if unsupported_endpoint is not None:
|
||||
raise unsupported_endpoint
|
||||
|
||||
# Merge model-level guardrails before pre_call_hook so DB/UI-configured
|
||||
# guardrails actually execute on pre_call. Without this, guardrails set
|
||||
# via litellm_params.guardrails are only honored on post_call paths
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, Any, Final, Literal
|
|||
|
||||
import httpx
|
||||
from fastapi import HTTPException, status
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
|
||||
|
||||
import litellm
|
||||
from litellm.proxy._types import ProxyException, UserAPIKeyAuth
|
||||
|
|
@ -332,6 +332,67 @@ def raise_if_required_body_param_missing(
|
|||
)
|
||||
|
||||
|
||||
CHAT_SHAPED_ENDPOINT_BY_ROUTE_TYPE: Final[Mapping[str, str]] = MappingProxyType(
|
||||
{
|
||||
"acompletion": "/v1/chat/completions",
|
||||
"anthropic_messages": "/v1/messages",
|
||||
"aresponses": "/v1/responses",
|
||||
}
|
||||
)
|
||||
|
||||
CHAT_SHAPED_ENDPOINTS: Final[frozenset[str]] = frozenset(
|
||||
{"/v1/chat/completions", "/chat/completions", "/v1/messages", "/v1/responses", "/responses"}
|
||||
)
|
||||
|
||||
CHAT_SHAPED_MODES: Final[frozenset[str]] = frozenset({"chat", "completion", "responses"})
|
||||
|
||||
|
||||
class _DeploymentEndpointDeclaration(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
mode: str | None = None
|
||||
supported_endpoints: tuple[str, ...] | None = None
|
||||
|
||||
|
||||
class ProxyUnsupportedEndpointError(ProxyException):
|
||||
pass
|
||||
|
||||
|
||||
def _declared_non_chat_endpoints(deployment: Mapping[str, object]) -> frozenset[str] | None:
|
||||
try:
|
||||
declared: Final = _DeploymentEndpointDeclaration.model_validate(deployment.get("model_info") or {})
|
||||
except ValidationError:
|
||||
return None
|
||||
if declared.mode is None or declared.mode in CHAT_SHAPED_MODES or declared.supported_endpoints is None:
|
||||
return None
|
||||
endpoints: Final = frozenset(declared.supported_endpoints)
|
||||
return endpoints if endpoints.isdisjoint(CHAT_SHAPED_ENDPOINTS) else None
|
||||
|
||||
|
||||
def endpoint_unsupported_by_model(
|
||||
route_type: str,
|
||||
model: object,
|
||||
llm_router: LitellmRouter | None,
|
||||
team_id: str | None,
|
||||
) -> ProxyUnsupportedEndpointError | None:
|
||||
endpoint: Final = CHAT_SHAPED_ENDPOINT_BY_ROUTE_TYPE.get(route_type)
|
||||
if endpoint is None or not isinstance(model, str) or not model or llm_router is None:
|
||||
return None
|
||||
deployments: Final = tuple(llm_router.get_model_list(model_name=model, team_id=team_id) or ())
|
||||
non_chat: Final = tuple(
|
||||
endpoints for deployment in deployments if (endpoints := _declared_non_chat_endpoints(deployment)) is not None
|
||||
)
|
||||
if not deployments or len(non_chat) < len(deployments):
|
||||
return None
|
||||
supported: Final = ", ".join(sorted(frozenset[str]().union(*non_chat))) or "none"
|
||||
return ProxyUnsupportedEndpointError(
|
||||
message=f"{endpoint} is not supported by model={model}. Supported endpoints: {supported}",
|
||||
type="invalid_request_error",
|
||||
param="model",
|
||||
code=status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
|
||||
|
||||
class MockTestingParamsDisabledError(HTTPException):
|
||||
def __init__(self, params: tuple[str, ...]):
|
||||
super().__init__(
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
- {id: llm.completions.openai.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: completions, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_completions_endpoint_e2e.py", rationale: "Legacy text /completions endpoint, second-highest production request volume"}
|
||||
- {id: llm.embeddings.openai.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: embeddings, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_embeddings_endpoint_e2e.py:23", rationale: "Core endpoint, live vector response"}
|
||||
- {id: llm.embeddings.openai.input_validation.nonstream.works, module: llm, tier: P1, subject_endpoint: embeddings, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], source: "vendor strategy §9.3 / LIT-4778", rationale: "Missing model/input on /embeddings return client errors"}
|
||||
- {id: llm.embeddings.openai.supported_endpoints_gate.nonstream.works, module: llm, tier: P1, subject_endpoint: embeddings, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], fail_before_fix: proven, source: "test_embedding_only_supported_endpoints_e2e.py", rationale: "A model whose every deployment lists only non-chat supported_endpoints answers each chat-shaped route 400 at the proxy, naming the endpoint and the declared list, while /embeddings keeps serving"}
|
||||
- {id: llm.embeddings.openai.basic.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: embeddings, route: openai, capability: basic, streaming: nonstream, assertions: [cost_logged], source: "SPEND_TRACKING_COVERAGE_MATRIX.md:34", rationale: "Cost tracking on embeddings"}
|
||||
- {id: llm.embeddings.openai.passthrough.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: embeddings, route: openai, capability: basic, streaming: nonstream, assertions: [cost_logged], source: "test_passthrough_e2e.py", rationale: "POST /openai_passthrough/v1/embeddings is costed; the route wrote no spend row at all, so budgets never saw traffic OpenAI was billing for (GitHub issue #36646)"}
|
||||
- {id: llm.embeddings.azure_openai.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: embeddings, route: azure_openai, capability: basic, streaming: nonstream, assertions: [works], source: "llms/azure/azure.py", rationale: "Azure embeddings via translation"}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,180 @@
|
|||
"""Live e2e: an embedding model whose deployments declare supported_endpoints refuses chat-shaped calls.
|
||||
|
||||
Each test registers an OpenAI embedding deployment whose model_info.supported_endpoints lists only
|
||||
/v1/embeddings, or a control deployment on the same backend that declares nothing. /embeddings serves
|
||||
real vectors on both. Every chat-shaped route to the declared model answers 400 with the proxy's own
|
||||
message, while the same chat call to the control goes to OpenAI and comes back with whatever OpenAI
|
||||
answers, so getting the proxy's message is the proof that no provider call was made.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
from typing import Final
|
||||
|
||||
import anthropic
|
||||
import openai
|
||||
import pytest
|
||||
from e2e_config import unique_marker
|
||||
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
|
||||
from lifecycle import ResourceManager
|
||||
from models import AnthropicErrorBody, AnthropicErrorEvent, LiteLLMParamsBody, ModelInfoBody, ModelNewBody
|
||||
from openai import OpenAI
|
||||
from proxy_client import ProxyClient
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
from sdk_clients import NO_PROXY_CACHE, SdkClients
|
||||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
||||
EMBEDDING_BACKEND: Final = "openai/text-embedding-3-small"
|
||||
EMBEDDINGS_ONLY: Final = ("/v1/embeddings",)
|
||||
PROMPT: Final = "Say this is a test!"
|
||||
UNSUPPORTED_MARKER: Final = "is not supported by model="
|
||||
|
||||
|
||||
class _OpenAiErrorDetail(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
message: str
|
||||
type: str
|
||||
param: str | None = None
|
||||
|
||||
|
||||
def _subject(route: Route) -> Subject:
|
||||
return Subject(
|
||||
domain=Domain.LLM_TRANSLATION,
|
||||
route=route,
|
||||
providers=(Provider.OPENAI,),
|
||||
models=(EMBEDDING_BACKEND,),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
|
||||
|
||||
def _register(
|
||||
proxy: ProxyClient, resources: ResourceManager, prefix: str, supported_endpoints: tuple[str, ...] | None
|
||||
) -> str:
|
||||
model: Final = f"{prefix}-{unique_marker()}"
|
||||
model_id: Final = proxy.register_model(
|
||||
ModelNewBody(
|
||||
model_name=model,
|
||||
litellm_params=LiteLLMParamsBody(model=EMBEDDING_BACKEND, api_key="os.environ/OPENAI_API_KEY"),
|
||||
model_info=ModelInfoBody(
|
||||
mode="embedding",
|
||||
supported_endpoints=None if supported_endpoints is None else list(supported_endpoints),
|
||||
),
|
||||
)
|
||||
)
|
||||
resources.defer(lambda: proxy.delete_model(model_id))
|
||||
return model
|
||||
|
||||
|
||||
def _openai_at(sdk: SdkClients, api_prefix: str, key: str) -> OpenAI:
|
||||
return dataclasses.replace(sdk, base_url=f"{sdk.base_url}{api_prefix}").openai(key)
|
||||
|
||||
|
||||
def _unsupported_message(endpoint: str, model: str) -> str:
|
||||
return f"{endpoint} {UNSUPPORTED_MARKER}{model}. Supported endpoints: {', '.join(sorted(EMBEDDINGS_ONLY))}"
|
||||
|
||||
|
||||
def _chat_error_message(client: OpenAI, model: str) -> str:
|
||||
try:
|
||||
_ = client.chat.completions.create(
|
||||
model=model, messages=[{"role": "user", "content": PROMPT}], extra_body=NO_PROXY_CACHE
|
||||
)
|
||||
except openai.APIStatusError as error:
|
||||
return f"{error.status_code} {error.message}"
|
||||
return ""
|
||||
|
||||
|
||||
def _assert_openai_rejection(error: openai.APIStatusError, endpoint: str, model: str) -> None:
|
||||
assert error.status_code == 400, f"expected the proxy's 400, got {error.status_code}: {error.message}"
|
||||
assert _OpenAiErrorDetail.model_validate(error.body) == _OpenAiErrorDetail(
|
||||
message=_unsupported_message(endpoint, model), type="invalid_request_error", param="model"
|
||||
), f"unexpected error body: {error.body!r}"
|
||||
|
||||
|
||||
class TestEmbeddingOnlySupportedEndpoints:
|
||||
@pytest.mark.covers("llm.embeddings.openai.supported_endpoints_gate.nonstream.works")
|
||||
@meta(_subject(Route.EMBEDDINGS))
|
||||
@pytest.mark.parametrize(
|
||||
("prefix", "supported_endpoints"),
|
||||
(
|
||||
pytest.param("e2e-embed-only", EMBEDDINGS_ONLY, id="embed-only"),
|
||||
pytest.param("e2e-embed-control", None, id="control"),
|
||||
),
|
||||
)
|
||||
def test_embeddings_return_a_vector(
|
||||
self,
|
||||
proxy: ProxyClient,
|
||||
resources: ResourceManager,
|
||||
sdk: SdkClients,
|
||||
prefix: str,
|
||||
supported_endpoints: tuple[str, ...] | None,
|
||||
) -> None:
|
||||
model: Final = _register(proxy, resources, prefix, supported_endpoints)
|
||||
embeddings: Final = sdk.openai(resources.key()).embeddings.create(
|
||||
model=model, input=PROMPT, extra_body=NO_PROXY_CACHE
|
||||
)
|
||||
assert embeddings.data, f"/embeddings returned no data for {model}: {embeddings!r}"
|
||||
assert any(component != 0.0 for component in embeddings.data[0].embedding), (
|
||||
f"/embeddings returned an empty or all-zero vector for {model}"
|
||||
)
|
||||
|
||||
@pytest.mark.covers("llm.embeddings.openai.supported_endpoints_gate.nonstream.works")
|
||||
@meta(_subject(Route.CHAT_COMPLETIONS))
|
||||
@pytest.mark.parametrize("api_prefix", ("/v1", ""), ids=("v1-chat-completions", "chat-completions"))
|
||||
def test_chat_completions_are_rejected_before_any_provider_call(
|
||||
self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients, api_prefix: str
|
||||
) -> None:
|
||||
model: Final = _register(proxy, resources, "e2e-embed-only", EMBEDDINGS_ONLY)
|
||||
with pytest.raises(openai.APIStatusError) as raised:
|
||||
_ = _openai_at(sdk, api_prefix, resources.key()).chat.completions.create(
|
||||
model=model, messages=[{"role": "user", "content": PROMPT}], extra_body=NO_PROXY_CACHE
|
||||
)
|
||||
_assert_openai_rejection(raised.value, "/v1/chat/completions", model)
|
||||
|
||||
@pytest.mark.covers("llm.embeddings.openai.supported_endpoints_gate.nonstream.works")
|
||||
@meta(_subject(Route.RESPONSES))
|
||||
@pytest.mark.parametrize("api_prefix", ("/v1", ""), ids=("v1-responses", "responses"))
|
||||
def test_responses_are_rejected_before_any_provider_call(
|
||||
self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients, api_prefix: str
|
||||
) -> None:
|
||||
model: Final = _register(proxy, resources, "e2e-embed-only", EMBEDDINGS_ONLY)
|
||||
with pytest.raises(openai.APIStatusError) as raised:
|
||||
_ = _openai_at(sdk, api_prefix, resources.key()).responses.create(
|
||||
model=model, input=PROMPT, extra_body=NO_PROXY_CACHE
|
||||
)
|
||||
_assert_openai_rejection(raised.value, "/v1/responses", model)
|
||||
|
||||
@pytest.mark.covers("llm.embeddings.openai.supported_endpoints_gate.nonstream.works")
|
||||
@meta(_subject(Route.MESSAGES))
|
||||
def test_messages_are_rejected_before_any_provider_call(
|
||||
self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients
|
||||
) -> None:
|
||||
model: Final = _register(proxy, resources, "e2e-embed-only", EMBEDDINGS_ONLY)
|
||||
with pytest.raises(anthropic.APIStatusError) as raised:
|
||||
_ = sdk.anthropic(resources.key()).messages.create(
|
||||
model=model,
|
||||
max_tokens=16,
|
||||
messages=[{"role": "user", "content": PROMPT}],
|
||||
extra_body=NO_PROXY_CACHE,
|
||||
)
|
||||
assert raised.value.status_code == 400, (
|
||||
f"expected the proxy's 400, got {raised.value.status_code}: {raised.value.message}"
|
||||
)
|
||||
assert AnthropicErrorEvent.model_validate(raised.value.body) == AnthropicErrorEvent(
|
||||
type="error",
|
||||
error=AnthropicErrorBody(type="invalid_request_error", message=_unsupported_message("/v1/messages", model)),
|
||||
), f"unexpected Anthropic error envelope: {raised.value.body!r}"
|
||||
|
||||
@pytest.mark.covers("llm.embeddings.openai.supported_endpoints_gate.nonstream.works")
|
||||
@meta(_subject(Route.CHAT_COMPLETIONS))
|
||||
def test_chat_completions_without_supported_endpoints_reach_the_provider(
|
||||
self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients
|
||||
) -> None:
|
||||
model: Final = _register(proxy, resources, "e2e-embed-control", None)
|
||||
error_message: Final = _chat_error_message(_openai_at(sdk, "/v1", resources.key()), model)
|
||||
assert UNSUPPORTED_MARKER not in error_message, (
|
||||
f"a model that declares no supported_endpoints must reach the provider, but the proxy "
|
||||
f"rejected it: {error_message}"
|
||||
)
|
||||
|
|
@ -1413,7 +1413,7 @@ class LiteLLMParamsBody(BaseModel):
|
|||
order: int | None = None
|
||||
|
||||
|
||||
ModelMode = Literal["chat", "batch", "realtime", "image_generation"]
|
||||
ModelMode = Literal["chat", "embedding", "batch", "realtime", "image_generation"]
|
||||
|
||||
|
||||
class ModelInfoBody(BaseModel):
|
||||
|
|
|
|||
|
|
@ -72,6 +72,7 @@ from litellm.proxy.hooks.model_max_budget_limiter import (
|
|||
model_budget_spend_cache_key,
|
||||
team_member_budget_entity_id,
|
||||
)
|
||||
from litellm.proxy.route_llm_request import ProxyUnsupportedEndpointError
|
||||
from litellm.proxy.utils import ProxyLogging
|
||||
from litellm.router import Router
|
||||
from litellm.router_utils.add_retry_fallback_headers import prepare_response_for_header_attachment
|
||||
|
|
@ -711,6 +712,67 @@ class TestProxyBaseLLMRequestProcessing:
|
|||
assert returned_data["metadata"]["tags"] == ["existing-tag"]
|
||||
tag_budget_check.assert_not_awaited()
|
||||
|
||||
@staticmethod
|
||||
def _router_with_embedding_only_model() -> Router:
|
||||
return Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "embed-only",
|
||||
"litellm_params": {"model": "openai/text-embedding-3-small", "api_key": "sk-test"},
|
||||
"model_info": {"mode": "embedding", "supported_endpoints": ["/v1/embeddings"]},
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_common_processing_pre_call_logic_rejects_chat_on_embedding_only_model_before_pre_call_hooks(
|
||||
self, monkeypatch
|
||||
):
|
||||
processing_obj, mock_request, mock_proxy_logging_obj, mock_proxy_config, _ = self._guardrail_tag_budget_harness(
|
||||
monkeypatch,
|
||||
request_body={"model": "embed-only", "messages": [{"role": "user", "content": "hello"}]},
|
||||
guardrail_tags=[],
|
||||
)
|
||||
|
||||
with pytest.raises(ProxyUnsupportedEndpointError) as exc_info:
|
||||
await processing_obj.common_processing_pre_call_logic(
|
||||
request=mock_request,
|
||||
general_settings={},
|
||||
user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-test"),
|
||||
proxy_logging_obj=mock_proxy_logging_obj,
|
||||
proxy_config=mock_proxy_config,
|
||||
route_type="acompletion",
|
||||
llm_router=self._router_with_embedding_only_model(),
|
||||
)
|
||||
|
||||
assert exc_info.value.code == "400"
|
||||
assert exc_info.value.message == (
|
||||
"/v1/chat/completions is not supported by model=embed-only. Supported endpoints: /v1/embeddings"
|
||||
)
|
||||
mock_proxy_logging_obj.pre_call_hook.assert_not_awaited()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_common_processing_pre_call_logic_lets_embeddings_through_on_embedding_only_model(self, monkeypatch):
|
||||
processing_obj, mock_request, mock_proxy_logging_obj, mock_proxy_config, _ = self._guardrail_tag_budget_harness(
|
||||
monkeypatch,
|
||||
request_body={"model": "embed-only", "input": ["hello"]},
|
||||
guardrail_tags=[],
|
||||
route="/v1/embeddings",
|
||||
)
|
||||
|
||||
returned_data, _ = await processing_obj.common_processing_pre_call_logic(
|
||||
request=mock_request,
|
||||
general_settings={},
|
||||
user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-test"),
|
||||
proxy_logging_obj=mock_proxy_logging_obj,
|
||||
proxy_config=mock_proxy_config,
|
||||
route_type="aembedding",
|
||||
llm_router=self._router_with_embedding_only_model(),
|
||||
)
|
||||
|
||||
assert returned_data["model"] == "embed-only"
|
||||
mock_proxy_logging_obj.pre_call_hook.assert_awaited_once()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_common_processing_pre_call_logic_skips_guardrail_tag_budget_check_for_zero_cost_model(
|
||||
self, monkeypatch
|
||||
|
|
|
|||
|
|
@ -7,7 +7,13 @@ from unittest.mock import MagicMock
|
|||
|
||||
from fastapi import HTTPException
|
||||
|
||||
from litellm.proxy.route_llm_request import ProxyModelNotFoundError, route_request
|
||||
from litellm import Router
|
||||
from litellm.proxy.route_llm_request import (
|
||||
ProxyModelNotFoundError,
|
||||
ProxyUnsupportedEndpointError,
|
||||
endpoint_unsupported_by_model,
|
||||
route_request,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
|
@ -1760,3 +1766,153 @@ async def test_route_request_router_settings_override_skips_null_fields():
|
|||
assert "context_window_fallbacks" not in call_kwargs
|
||||
assert "num_retries" not in call_kwargs
|
||||
assert "model_group_retry_policy" not in call_kwargs
|
||||
|
||||
|
||||
CHAT_SHAPED_ROUTES: Final = (
|
||||
("acompletion", "/v1/chat/completions"),
|
||||
("anthropic_messages", "/v1/messages"),
|
||||
("aresponses", "/v1/responses"),
|
||||
)
|
||||
|
||||
CHAT_SHAPED_ROUTE_TYPES: Final = tuple(route_type for route_type, _ in CHAT_SHAPED_ROUTES)
|
||||
|
||||
EMBEDDING_ONLY: Final = {"mode": "embedding", "supported_endpoints": ["/v1/embeddings"]}
|
||||
|
||||
|
||||
def _embedding_deployment(model_name: str, model_info: dict[str, object]) -> dict[str, object]:
|
||||
return {
|
||||
"model_name": model_name,
|
||||
"litellm_params": {"model": "sagemaker/openai/embed-endpoint", "aws_region_name": "us-east-1"},
|
||||
"model_info": model_info,
|
||||
}
|
||||
|
||||
|
||||
def _router_with(*deployments: dict[str, object], **router_kwargs: object) -> Router:
|
||||
return Router(model_list=list(deployments), **router_kwargs)
|
||||
|
||||
|
||||
def _unsupported(route_type: str, model: object, router: Router | None) -> ProxyUnsupportedEndpointError | None:
|
||||
return endpoint_unsupported_by_model(route_type=route_type, model=model, llm_router=router, team_id=None)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("route_type, endpoint", CHAT_SHAPED_ROUTES)
|
||||
def test_endpoint_unsupported_rejects_chat_routes_on_embedding_only_model(route_type, endpoint):
|
||||
error: Final = _unsupported(
|
||||
route_type, "embed-only", _router_with(_embedding_deployment("embed-only", EMBEDDING_ONLY))
|
||||
)
|
||||
|
||||
assert isinstance(error, ProxyUnsupportedEndpointError)
|
||||
assert error.code == "400"
|
||||
assert error.param == "model"
|
||||
assert error.type == "invalid_request_error"
|
||||
assert error.message == f"{endpoint} is not supported by model=embed-only. Supported endpoints: /v1/embeddings"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("route_type", ["aembedding", "atext_completion", "aimage_generation"])
|
||||
def test_endpoint_unsupported_leaves_non_chat_routes_alone(route_type):
|
||||
router: Final = _router_with(_embedding_deployment("embed-only", EMBEDDING_ONLY))
|
||||
|
||||
assert _unsupported(route_type, "embed-only", router) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("route_type", CHAT_SHAPED_ROUTE_TYPES)
|
||||
@pytest.mark.parametrize(
|
||||
"model_info",
|
||||
[
|
||||
{"mode": "embedding"},
|
||||
{},
|
||||
{"mode": "embedding", "supported_endpoints": None},
|
||||
{"mode": "embedding", "supported_endpoints": "/v1/embeddings"},
|
||||
{"mode": 1, "supported_endpoints": ["/v1/embeddings"]},
|
||||
],
|
||||
)
|
||||
def test_endpoint_unsupported_keeps_todays_behavior_without_a_valid_declaration(route_type, model_info):
|
||||
router: Final = _router_with(_embedding_deployment("embed-model", model_info))
|
||||
|
||||
assert _unsupported(route_type, "embed-model", router) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("route_type", CHAT_SHAPED_ROUTE_TYPES)
|
||||
@pytest.mark.parametrize(
|
||||
"model_info",
|
||||
[
|
||||
{"supported_endpoints": ["/v1/embeddings"]},
|
||||
{"mode": "chat", "supported_endpoints": ["/v1/batch"]},
|
||||
{"mode": "completion", "supported_endpoints": ["/fine_tuning"]},
|
||||
{"mode": "responses", "supported_endpoints": ["/v1/realtime"]},
|
||||
],
|
||||
)
|
||||
def test_endpoint_unsupported_needs_a_non_chat_mode_to_reject(route_type, model_info):
|
||||
router: Final = _router_with(_embedding_deployment("advisory", model_info))
|
||||
|
||||
assert _unsupported(route_type, "advisory", router) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("route_type", CHAT_SHAPED_ROUTE_TYPES)
|
||||
@pytest.mark.parametrize(
|
||||
"declared",
|
||||
[["/v1/messages"], ["/v1/responses"], ["/v1/chat/completions"], ["/chat/completions"], ["/responses"]],
|
||||
)
|
||||
def test_endpoint_unsupported_allows_every_chat_route_when_any_chat_endpoint_is_declared(route_type, declared):
|
||||
router: Final = _router_with(
|
||||
_embedding_deployment("chat-model", {"mode": "embedding", "supported_endpoints": declared})
|
||||
)
|
||||
|
||||
assert _unsupported(route_type, "chat-model", router) is None
|
||||
|
||||
|
||||
def test_endpoint_unsupported_allows_a_group_with_one_deployment_that_declares_nothing():
|
||||
router: Final = _router_with(
|
||||
_embedding_deployment("mixed", EMBEDDING_ONLY),
|
||||
_embedding_deployment("mixed", {"mode": "embedding"}),
|
||||
)
|
||||
|
||||
assert _unsupported("acompletion", "mixed", router) is None
|
||||
|
||||
|
||||
def test_endpoint_unsupported_names_every_endpoint_the_group_declares():
|
||||
router: Final = _router_with(
|
||||
_embedding_deployment("embed-group", EMBEDDING_ONLY),
|
||||
_embedding_deployment(
|
||||
"embed-group", {"mode": "embedding", "supported_endpoints": ["/v1/rerank", "/v1/embeddings"]}
|
||||
),
|
||||
)
|
||||
|
||||
error: Final = _unsupported("acompletion", "embed-group", router)
|
||||
|
||||
assert isinstance(error, ProxyUnsupportedEndpointError)
|
||||
assert error.message == (
|
||||
"/v1/chat/completions is not supported by model=embed-group. Supported endpoints: /v1/embeddings, /v1/rerank"
|
||||
)
|
||||
|
||||
|
||||
def test_endpoint_unsupported_says_none_for_an_empty_declared_list():
|
||||
router: Final = _router_with(_embedding_deployment("nothing", {"mode": "embedding", "supported_endpoints": []}))
|
||||
|
||||
error: Final = _unsupported("aresponses", "nothing", router)
|
||||
|
||||
assert isinstance(error, ProxyUnsupportedEndpointError)
|
||||
assert error.message == "/v1/responses is not supported by model=nothing. Supported endpoints: none"
|
||||
|
||||
|
||||
def test_endpoint_unsupported_follows_a_model_group_alias():
|
||||
router: Final = _router_with(
|
||||
_embedding_deployment("embed-only", EMBEDDING_ONLY),
|
||||
model_group_alias={"embed-alias": "embed-only"},
|
||||
)
|
||||
|
||||
error: Final = _unsupported("anthropic_messages", "embed-alias", router)
|
||||
|
||||
assert isinstance(error, ProxyUnsupportedEndpointError)
|
||||
assert error.message == "/v1/messages is not supported by model=embed-alias. Supported endpoints: /v1/embeddings"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["unknown-model", "", None])
|
||||
def test_endpoint_unsupported_ignores_models_without_deployments(model):
|
||||
router: Final = _router_with(_embedding_deployment("embed-only", EMBEDDING_ONLY))
|
||||
|
||||
assert _unsupported("acompletion", model, router) is None
|
||||
|
||||
|
||||
def test_endpoint_unsupported_skips_the_check_without_a_router():
|
||||
assert _unsupported("acompletion", "embed-only", None) is None
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue