mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-29 06:33:28 +00:00
fix(proxy): only step aside for fallbacks that can answer chat, gate the Responses WebSocket after alias resolution, and compare declarations to the loaded catalog
This commit is contained in:
mateo-berri
parent
0264476d7f
commit
6728e9562d
6 changed files with 178 additions and 72 deletions
|
|
@ -36,7 +36,7 @@ from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # leg
|
|||
read_request_body,
|
||||
safe_set_request_parsed_body,
|
||||
)
|
||||
from litellm.proxy.route_llm_request import endpoint_unsupported_by_model, raise_if_required_body_param_missing
|
||||
from litellm.proxy.route_llm_request import raise_if_required_body_param_missing
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
from litellm.types.llms.openai import (
|
||||
REASONING_EFFORT,
|
||||
|
|
@ -1546,14 +1546,6 @@ async def _responses_websocket_session(
|
|||
user_api_key_dict=user_api_key_dict,
|
||||
llm_router=llm_router,
|
||||
)
|
||||
unsupported_endpoint: Final = endpoint_unsupported_by_model(
|
||||
route_type="aresponses",
|
||||
data=data,
|
||||
llm_router=llm_router,
|
||||
team_id=user_api_key_dict.team_id,
|
||||
)
|
||||
if unsupported_endpoint is not None:
|
||||
raise unsupported_endpoint
|
||||
(
|
||||
data,
|
||||
litellm_logging_obj,
|
||||
|
|
|
|||
|
|
@ -374,6 +374,7 @@ CHAT_SHAPED_ENDPOINT_BY_ROUTE_TYPE: Final[Mapping[str, str]] = MappingProxyType(
|
|||
"agenerate_content": "generateContent",
|
||||
"agenerate_content_stream": "streamGenerateContent",
|
||||
"acreate_interaction": "/v1beta/interactions",
|
||||
"_aresponses_websocket": "/v1/responses",
|
||||
}
|
||||
)
|
||||
|
||||
|
|
@ -390,8 +391,6 @@ CHAT_SHAPED_ENDPOINTS: Final[frozenset[str]] = frozenset(
|
|||
}
|
||||
)
|
||||
|
||||
DECLARATION_FIELDS: Final[frozenset[str]] = frozenset({"mode", "supported_endpoints"})
|
||||
|
||||
CHAT_SHAPED_MODES: Final[frozenset[str]] = frozenset({"chat", "completion", "responses"})
|
||||
|
||||
|
||||
|
|
@ -411,32 +410,18 @@ _FALLBACK_LIST_ADAPTER: Final[TypeAdapter[list[object]]] = TypeAdapter( # mutab
|
|||
)
|
||||
|
||||
|
||||
class _LiveCostMap(Protocol):
|
||||
@property
|
||||
def model_cost(self) -> Mapping[str, object]: ...
|
||||
|
||||
|
||||
class _RouterFallbacks(Protocol):
|
||||
@property
|
||||
def fallbacks(self) -> Sequence[object] | None: ...
|
||||
|
||||
|
||||
def _live_cost_map_entry(catalog: _LiveCostMap, catalog_key: str) -> Mapping[str, object]:
|
||||
try:
|
||||
return JSON_OBJECT_ADAPTER.validate_python(catalog.model_cost.get(catalog_key) or {})
|
||||
except ValidationError:
|
||||
return MappingProxyType({})
|
||||
|
||||
|
||||
def _echoed_from_cost_map(model_info: Mapping[str, object]) -> frozenset[str]:
|
||||
catalog_key: Final = model_info.get(COST_MAP_LOOKUP_KEY)
|
||||
if not isinstance(catalog_key, str):
|
||||
return frozenset()
|
||||
return frozenset(
|
||||
echoed_cost_map_fields(
|
||||
model_info,
|
||||
_live_cost_map_entry(litellm, catalog_key),
|
||||
GetModelCostMap.loaded_model_cost_map().get(catalog_key) or MappingProxyType({}),
|
||||
model_info, GetModelCostMap.loaded_model_cost_map().get(catalog_key) or MappingProxyType({})
|
||||
)
|
||||
)
|
||||
|
||||
|
|
@ -447,13 +432,21 @@ def _declared_non_chat_endpoints(deployment: Mapping[str, object]) -> frozenset[
|
|||
declared: Final = _DeploymentEndpointDeclaration.model_validate(model_info)
|
||||
except ValidationError:
|
||||
return None
|
||||
if not DECLARATION_FIELDS.isdisjoint(_echoed_from_cost_map(model_info)):
|
||||
return None
|
||||
mode: Final = (declared.mode or "").strip().lower()
|
||||
if not mode or mode in CHAT_SHAPED_MODES or declared.supported_endpoints is None:
|
||||
return None
|
||||
endpoints: Final = frozenset(declared.supported_endpoints)
|
||||
return endpoints if endpoints.isdisjoint(CHAT_SHAPED_ENDPOINTS) else None
|
||||
if not endpoints.isdisjoint(CHAT_SHAPED_ENDPOINTS):
|
||||
return None
|
||||
return None if "supported_endpoints" in _echoed_from_cost_map(model_info) else endpoints
|
||||
|
||||
|
||||
def _group_deployments(model: str, llm_router: LitellmRouter, team_id: str | None) -> tuple[Mapping[str, object], ...]:
|
||||
return tuple(llm_router.get_model_list(model_name=model, team_id=team_id) or ())
|
||||
|
||||
|
||||
def _any_serves_chat(deployments: Sequence[Mapping[str, object]]) -> bool:
|
||||
return any(_declared_non_chat_endpoints(deployment) is None for deployment in deployments)
|
||||
|
||||
|
||||
def _override_fallbacks(data: Mapping[str, object]) -> object:
|
||||
|
|
@ -474,22 +467,65 @@ def _router_fallbacks(llm_router: _RouterFallbacks) -> Sequence[object] | None:
|
|||
return llm_router.fallbacks
|
||||
|
||||
|
||||
def _fallback_would_answer(model: str, data: Mapping[str, object], llm_router: LitellmRouter) -> bool:
|
||||
def _fallback_targets(
|
||||
model: str,
|
||||
data: Mapping[str, object],
|
||||
fallbacks: list[object], # mutable-ok: mirrors get_fallback_model_group's contract
|
||||
) -> Sequence[object]:
|
||||
if check_non_standard_fallback_format(fallbacks=fallbacks):
|
||||
return fallbacks
|
||||
resolved, _ = get_fallback_model_group_for_lookup_groups(
|
||||
fallbacks=fallbacks,
|
||||
lookup_groups=fallback_lookup_groups(data, model),
|
||||
)
|
||||
return resolved or ()
|
||||
|
||||
|
||||
def _fallback_target_group(target: object) -> str | None:
|
||||
if isinstance(target, str):
|
||||
return target
|
||||
try:
|
||||
entry: Final = JSON_OBJECT_ADAPTER.validate_python(target)
|
||||
except ValidationError:
|
||||
return None
|
||||
named_model: Final = entry.get("model")
|
||||
return named_model if isinstance(named_model, str) else None
|
||||
|
||||
|
||||
def _fallback_target_answers(
|
||||
group: str | None,
|
||||
data: Mapping[str, object],
|
||||
llm_router: LitellmRouter,
|
||||
team_id: str | None,
|
||||
walked: frozenset[str],
|
||||
) -> bool:
|
||||
if group is None or group in walked:
|
||||
return False
|
||||
deployments: Final = _group_deployments(group, llm_router, team_id)
|
||||
if not deployments:
|
||||
return False
|
||||
return _any_serves_chat(deployments) or _fallback_would_answer(group, data, llm_router, team_id, walked)
|
||||
|
||||
|
||||
def _fallback_would_answer(
|
||||
model: str,
|
||||
data: Mapping[str, object],
|
||||
llm_router: LitellmRouter,
|
||||
team_id: str | None,
|
||||
walked: frozenset[str] = frozenset(),
|
||||
) -> bool:
|
||||
if fallbacks_disabled_for_request(data):
|
||||
return False
|
||||
try:
|
||||
fallbacks: Final = _FALLBACK_LIST_ADAPTER.validate_python(_fallbacks_for_request(data, llm_router))
|
||||
except ValidationError:
|
||||
return False
|
||||
if not fallbacks:
|
||||
return False
|
||||
if check_non_standard_fallback_format(fallbacks=fallbacks):
|
||||
return True
|
||||
resolved, _ = get_fallback_model_group_for_lookup_groups(
|
||||
fallbacks=fallbacks,
|
||||
lookup_groups=fallback_lookup_groups(data, model),
|
||||
walked_here: Final = walked | {model}
|
||||
return any(
|
||||
has_unattempted_fallback_target((target,), data)
|
||||
and _fallback_target_answers(_fallback_target_group(target), data, llm_router, team_id, walked_here)
|
||||
for target in _fallback_targets(model, data, fallbacks)
|
||||
)
|
||||
return has_unattempted_fallback_target(resolved, data)
|
||||
|
||||
|
||||
def endpoint_unsupported_by_model(
|
||||
|
|
@ -502,15 +538,16 @@ def endpoint_unsupported_by_model(
|
|||
model: Final = data.get("model")
|
||||
if endpoint is None or not isinstance(model, str) or not model or llm_router is None:
|
||||
return None
|
||||
deployments: Final = tuple(llm_router.get_model_list(model_name=model, team_id=team_id) or ())
|
||||
non_chat: Final = tuple(
|
||||
endpoints for deployment in deployments if (endpoints := _declared_non_chat_endpoints(deployment)) is not None
|
||||
)
|
||||
if not deployments or len(non_chat) < len(deployments) or _fallback_would_answer(model, data, llm_router):
|
||||
deployments: Final = _group_deployments(model, llm_router, team_id)
|
||||
if not deployments or _any_serves_chat(deployments):
|
||||
return None
|
||||
supported: Final = ", ".join(sorted(frozenset[str]().union(*non_chat))) or "none"
|
||||
if _fallback_would_answer(model, data, llm_router, team_id):
|
||||
return None
|
||||
supported: Final = ", ".join(
|
||||
sorted(frozenset[str]().union(*(_declared_non_chat_endpoints(d) or frozenset() for d in deployments)))
|
||||
)
|
||||
return ProxyUnsupportedEndpointError(
|
||||
message=f"{endpoint} is not supported by model={model}. Supported endpoints: {supported}",
|
||||
message=f"{endpoint} is not supported by model={model}. Supported endpoints: {supported or 'none'}",
|
||||
type="invalid_request_error",
|
||||
param="model",
|
||||
code=status.HTTP_400_BAD_REQUEST,
|
||||
|
|
|
|||
|
|
@ -613,6 +613,16 @@ def test_an_empty_request_fallback_list_keeps_the_rejection(rig: _Rig) -> None:
|
|||
_assert_rejected(rig, response, marker, _openai_error("/v1/chat/completions", model, "/v1/embeddings"))
|
||||
|
||||
|
||||
def test_fallbacks_that_cannot_answer_chat_keep_the_rejection(rig: _Rig) -> None:
|
||||
model: Final = rig.names["embedding_only"]
|
||||
marker: Final = _marker()
|
||||
fallbacks: Final = [rig.names["image_generation"], model, f"gate-missing-{uuid.uuid4().hex}"]
|
||||
response: Final = rig.gateway.request(
|
||||
"POST", "/v1/chat/completions", {**_chat_body(model, marker), "fallbacks": fallbacks}
|
||||
)
|
||||
_assert_rejected(rig, response, marker, _openai_error("/v1/chat/completions", model, "/v1/embeddings"))
|
||||
|
||||
|
||||
def test_request_fallbacks_do_not_rescue_an_unknown_primary(rig: _Rig) -> None:
|
||||
marker: Final = _marker()
|
||||
response: Final = rig.gateway.request(
|
||||
|
|
|
|||
|
|
@ -1015,12 +1015,13 @@ class TestResponsesWSDeclaredNonChatModel:
|
|||
({"mode": "embedding"}, False),
|
||||
],
|
||||
)
|
||||
async def test_endpoint_rejects_a_model_declared_non_chat_before_routing(
|
||||
async def test_session_closes_on_a_model_declared_non_chat_before_routing(
|
||||
self, query_model: str | None, model_info: dict[str, object], rejected: bool
|
||||
):
|
||||
from litellm.proxy.response_api_endpoints.endpoints import (
|
||||
responses_websocket_endpoint,
|
||||
)
|
||||
from litellm.proxy.route_llm_request import endpoint_unsupported_by_model
|
||||
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
|
|
@ -1043,10 +1044,16 @@ class TestResponsesWSDeclaredNonChatModel:
|
|||
ws.send_text = AsyncMock()
|
||||
ws.close = AsyncMock()
|
||||
|
||||
async def gated_pre_call(**kwargs):
|
||||
unsupported = endpoint_unsupported_by_model(
|
||||
route_type=kwargs["route_type"], data={"model": kwargs["model"]}, llm_router=router, team_id=None
|
||||
)
|
||||
if unsupported is not None:
|
||||
raise unsupported
|
||||
return {"model": kwargs["model"], "litellm_metadata": {}}, MagicMock()
|
||||
|
||||
processor = MagicMock()
|
||||
processor.common_processing_pre_call_logic = AsyncMock(
|
||||
return_value=({"model": "embed-only", "litellm_metadata": {}}, MagicMock())
|
||||
)
|
||||
processor.common_processing_pre_call_logic = AsyncMock(side_effect=gated_pre_call)
|
||||
|
||||
async def fake_llm_call():
|
||||
return None
|
||||
|
|
@ -1084,7 +1091,6 @@ class TestResponsesWSDeclaredNonChatModel:
|
|||
"/v1/responses is not supported by model=embed-only. Supported endpoints: /v1/embeddings"
|
||||
)
|
||||
ws.close.assert_awaited_once_with(code=1008, reason="Pre-call error")
|
||||
processor.common_processing_pre_call_logic.assert_not_awaited()
|
||||
mock_route_request.assert_not_awaited()
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -725,12 +725,20 @@ class TestProxyBaseLLMRequestProcessing:
|
|||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"route_type, endpoint",
|
||||
[("acompletion", "/v1/chat/completions"), ("_aresponses_websocket", "/v1/responses")],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"requested_model, key_aliases",
|
||||
[("embed-only", {}), ("embed-alias", {"embed-alias": "embed-only"})],
|
||||
)
|
||||
async def test_common_processing_pre_call_logic_rejects_chat_on_embedding_only_model_before_pre_call_hooks(
|
||||
self, monkeypatch
|
||||
self, monkeypatch, route_type, endpoint, requested_model, key_aliases
|
||||
):
|
||||
processing_obj, mock_request, mock_proxy_logging_obj, mock_proxy_config, _ = self._guardrail_tag_budget_harness(
|
||||
monkeypatch,
|
||||
request_body={"model": "embed-only", "messages": [{"role": "user", "content": "hello"}]},
|
||||
request_body={"model": requested_model, "messages": [{"role": "user", "content": "hello"}]},
|
||||
guardrail_tags=[],
|
||||
)
|
||||
|
||||
|
|
@ -738,16 +746,16 @@ class TestProxyBaseLLMRequestProcessing:
|
|||
await processing_obj.common_processing_pre_call_logic(
|
||||
request=mock_request,
|
||||
general_settings={},
|
||||
user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-test"),
|
||||
user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-test", aliases=key_aliases),
|
||||
proxy_logging_obj=mock_proxy_logging_obj,
|
||||
proxy_config=mock_proxy_config,
|
||||
route_type="acompletion",
|
||||
route_type=route_type,
|
||||
llm_router=self._router_with_embedding_only_model(),
|
||||
)
|
||||
|
||||
assert exc_info.value.code == "400"
|
||||
assert exc_info.value.message == (
|
||||
"/v1/chat/completions is not supported by model=embed-only. Supported endpoints: /v1/embeddings"
|
||||
f"{endpoint} is not supported by model=embed-only. Supported endpoints: /v1/embeddings"
|
||||
)
|
||||
mock_proxy_logging_obj.pre_call_hook.assert_not_awaited()
|
||||
|
||||
|
|
|
|||
|
|
@ -3,13 +3,16 @@ import pytest
|
|||
|
||||
from types import SimpleNamespace
|
||||
from typing import Final
|
||||
from unittest.mock import MagicMock, patch
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from fastapi import HTTPException
|
||||
|
||||
import litellm
|
||||
from litellm import Router
|
||||
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
|
||||
from litellm.proxy.route_llm_request import (
|
||||
CHAT_SHAPED_ENDPOINTS,
|
||||
CHAT_SHAPED_MODES,
|
||||
ProxyModelNotFoundError,
|
||||
ProxyUnsupportedEndpointError,
|
||||
endpoint_unsupported_by_model,
|
||||
|
|
@ -1777,6 +1780,7 @@ CHAT_SHAPED_ROUTES: Final = (
|
|||
("agenerate_content", "generateContent"),
|
||||
("agenerate_content_stream", "streamGenerateContent"),
|
||||
("acreate_interaction", "/v1beta/interactions"),
|
||||
("_aresponses_websocket", "/v1/responses"),
|
||||
)
|
||||
|
||||
CHAT_SHAPED_ROUTE_TYPES: Final = tuple(route_type for route_type, _ in CHAT_SHAPED_ROUTES)
|
||||
|
|
@ -1817,7 +1821,7 @@ def test_endpoint_unsupported_rejects_chat_routes_on_embedding_only_model(route_
|
|||
assert error.message == f"{endpoint} is not supported by model=embed-only. Supported endpoints: /v1/embeddings"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("route_type", ["aembedding", "aimage_generation", "_aresponses_websocket"])
|
||||
@pytest.mark.parametrize("route_type", ["aembedding", "aimage_generation", "aspeech", "_arealtime"])
|
||||
def test_endpoint_unsupported_leaves_non_chat_routes_alone(route_type):
|
||||
router: Final = _router_with(_embedding_deployment("embed-only", EMBEDDING_ONLY))
|
||||
|
||||
|
|
@ -1944,6 +1948,8 @@ CHAT_FALLBACK_DEPLOYMENT: Final = {
|
|||
"litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "test-key"},
|
||||
}
|
||||
|
||||
EMBEDDING_FALLBACK_DEPLOYMENT: Final = _embedding_deployment("embed-fallback", EMBEDDING_ONLY)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"router_kwargs, request_kwargs",
|
||||
|
|
@ -1953,11 +1959,18 @@ CHAT_FALLBACK_DEPLOYMENT: Final = {
|
|||
({}, {"fallbacks": ["chat-fallback"]}),
|
||||
({}, {"fallbacks": [{"embed-only": ["chat-fallback"]}]}),
|
||||
({}, {"router_settings_override": {"fallbacks": [{"embed-only": ["chat-fallback"]}]}}),
|
||||
({"fallbacks": [{"embed-only": ["embed-fallback", "missing-group", "chat-fallback"]}]}, {}),
|
||||
({}, {"fallbacks": [{"model": "chat-fallback", "temperature": 0.2}]}),
|
||||
({"fallbacks": [{"embed-only": ["chat-alias"]}], "model_group_alias": {"chat-alias": "chat-fallback"}}, {}),
|
||||
({"fallbacks": [{"embed-only": ["embed-fallback"]}, {"embed-fallback": ["chat-fallback"]}]}, {}),
|
||||
],
|
||||
)
|
||||
def test_endpoint_unsupported_lets_a_configured_fallback_answer(router_kwargs, request_kwargs):
|
||||
router: Final = _router_with(
|
||||
_embedding_deployment("embed-only", EMBEDDING_ONLY), CHAT_FALLBACK_DEPLOYMENT, **router_kwargs
|
||||
_embedding_deployment("embed-only", EMBEDDING_ONLY),
|
||||
CHAT_FALLBACK_DEPLOYMENT,
|
||||
EMBEDDING_FALLBACK_DEPLOYMENT,
|
||||
**router_kwargs,
|
||||
)
|
||||
|
||||
assert _unsupported("acompletion", "embed-only", router, **request_kwargs) is None
|
||||
|
|
@ -1974,35 +1987,75 @@ def test_endpoint_unsupported_lets_a_configured_fallback_answer(router_kwargs, r
|
|||
{"router_settings_override": {"fallbacks": [{"other-group": ["chat-fallback"]}]}},
|
||||
),
|
||||
({}, {"fallbacks": [], "router_settings_override": {"fallbacks": [{"embed-only": ["chat-fallback"]}]}}),
|
||||
({"fallbacks": [{"embed-only": ["embed-fallback"]}]}, {}),
|
||||
({"fallbacks": [{"embed-only": ["embed-only"]}]}, {}),
|
||||
({"fallbacks": [{"embed-only": ["missing-group"]}]}, {}),
|
||||
({"default_fallbacks": ["embed-fallback"]}, {}),
|
||||
({}, {"fallbacks": ["embed-fallback", "missing-group"]}),
|
||||
({}, {"fallbacks": [{"model": "embed-fallback", "temperature": 0.2}]}),
|
||||
({"fallbacks": [{"embed-only": ["embed-alias"]}], "model_group_alias": {"embed-alias": "embed-fallback"}}, {}),
|
||||
({"fallbacks": [{"embed-only": ["embed-fallback"]}, {"embed-fallback": ["embed-only"]}]}, {}),
|
||||
({"fallbacks": [{"embed-only": ["embed-fallback"]}, {"embed-fallback": ["chat-fallback"]}]}, {"fallbacks": []}),
|
||||
],
|
||||
)
|
||||
def test_endpoint_unsupported_rejects_when_no_fallback_would_answer(router_kwargs, request_kwargs):
|
||||
router: Final = _router_with(
|
||||
_embedding_deployment("embed-only", EMBEDDING_ONLY), CHAT_FALLBACK_DEPLOYMENT, **router_kwargs
|
||||
_embedding_deployment("embed-only", EMBEDDING_ONLY),
|
||||
CHAT_FALLBACK_DEPLOYMENT,
|
||||
EMBEDDING_FALLBACK_DEPLOYMENT,
|
||||
**router_kwargs,
|
||||
)
|
||||
|
||||
assert isinstance(_unsupported("acompletion", "embed-only", router, **request_kwargs), ProxyUnsupportedEndpointError)
|
||||
assert isinstance(
|
||||
_unsupported("acompletion", "embed-only", router, **request_kwargs), ProxyUnsupportedEndpointError
|
||||
)
|
||||
|
||||
|
||||
ECHOED_CATALOG_ENTRY: Final = {"mode": "image_generation", "supported_endpoints": ["/v1/images/generations"]}
|
||||
def _catalog_key_declaring_non_chat() -> str:
|
||||
return min(
|
||||
key
|
||||
for key, entry in GetModelCostMap.loaded_model_cost_map().items()
|
||||
if entry.get("litellm_provider") == "openai"
|
||||
and entry.get("mode") not in CHAT_SHAPED_MODES
|
||||
and isinstance(endpoints := entry.get("supported_endpoints"), list)
|
||||
and CHAT_SHAPED_ENDPOINTS.isdisjoint(endpoints)
|
||||
)
|
||||
|
||||
|
||||
def _catalog_declaration(catalog_key: str) -> dict[str, object]:
|
||||
entry: Final = GetModelCostMap.loaded_model_cost_map()[catalog_key]
|
||||
return {"mode": entry["mode"], "supported_endpoints": entry["supported_endpoints"]}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_info, rejected",
|
||||
"declaration, with_key, rejected",
|
||||
[
|
||||
({"key": "echoed-image-model", **ECHOED_CATALOG_ENTRY}, False),
|
||||
({"key": "echoed-image-model", "mode": "image_generation", "supported_endpoints": ["/v1/embeddings"]}, False),
|
||||
({"key": "echoed-image-model", "mode": "embedding", "supported_endpoints": ["/v1/embeddings"]}, True),
|
||||
(dict(ECHOED_CATALOG_ENTRY), True),
|
||||
pytest.param("catalog", True, False, id="echo-of-the-catalog-row"),
|
||||
pytest.param("catalog-mode-typed-endpoints", True, True, id="endpoints-typed-over-an-echo"),
|
||||
pytest.param("typed-mode-catalog-endpoints", True, False, id="endpoints-left-as-echoed"),
|
||||
pytest.param("typed", True, True, id="both-typed-over-an-echo"),
|
||||
pytest.param("catalog", False, True, id="catalog-values-typed-by-hand"),
|
||||
],
|
||||
)
|
||||
def test_endpoint_unsupported_ignores_fields_copied_from_the_cost_map(model_info, rejected):
|
||||
router: Final = _router_with(_embedding_deployment("image-row", model_info))
|
||||
def test_endpoint_unsupported_ignores_endpoints_copied_from_the_loaded_catalog(declaration, with_key, rejected):
|
||||
catalog_key: Final = _catalog_key_declaring_non_chat()
|
||||
catalog: Final = _catalog_declaration(catalog_key)
|
||||
declared: Final = {
|
||||
"catalog": catalog,
|
||||
"catalog-mode-typed-endpoints": {**catalog, "supported_endpoints": ["/v1/embeddings"]},
|
||||
"typed-mode-catalog-endpoints": {**catalog, "mode": "embedding"},
|
||||
"typed": EMBEDDING_ONLY,
|
||||
}[declaration]
|
||||
router: Final = _router_with(
|
||||
{
|
||||
"model_name": "catalog-row",
|
||||
"litellm_params": {"model": catalog_key, "api_key": "test-key"},
|
||||
"model_info": {**({"key": catalog_key} if with_key else {}), **declared},
|
||||
}
|
||||
)
|
||||
|
||||
with patch.dict(litellm.model_cost, {"echoed-image-model": ECHOED_CATALOG_ENTRY}):
|
||||
error: Final = _unsupported("acompletion", "image-row", router)
|
||||
|
||||
assert isinstance(error, ProxyUnsupportedEndpointError) is rejected
|
||||
assert litellm.model_cost[catalog_key]["supported_endpoints"] == declared["supported_endpoints"]
|
||||
assert isinstance(_unsupported("acompletion", "catalog-row", router), ProxyUnsupportedEndpointError) is rejected
|
||||
|
||||
|
||||
def _router_with_defaults(default_litellm_params: dict[str, object]):
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue