diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index c6d5221c275..7f9fcca369b 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -36,7 +36,7 @@ from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # leg read_request_body, safe_set_request_parsed_body, ) -from litellm.proxy.route_llm_request import endpoint_unsupported_by_model, raise_if_required_body_param_missing +from litellm.proxy.route_llm_request import raise_if_required_body_param_missing from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.llms.openai import ( REASONING_EFFORT, @@ -1546,14 +1546,6 @@ async def _responses_websocket_session( user_api_key_dict=user_api_key_dict, llm_router=llm_router, ) - unsupported_endpoint: Final = endpoint_unsupported_by_model( - route_type="aresponses", - data=data, - llm_router=llm_router, - team_id=user_api_key_dict.team_id, - ) - if unsupported_endpoint is not None: - raise unsupported_endpoint ( data, litellm_logging_obj, diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index 087a7f56d7a..2f66327a8b9 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -374,6 +374,7 @@ CHAT_SHAPED_ENDPOINT_BY_ROUTE_TYPE: Final[Mapping[str, str]] = MappingProxyType( "agenerate_content": "generateContent", "agenerate_content_stream": "streamGenerateContent", "acreate_interaction": "/v1beta/interactions", + "_aresponses_websocket": "/v1/responses", } ) @@ -390,8 +391,6 @@ CHAT_SHAPED_ENDPOINTS: Final[frozenset[str]] = frozenset( } ) -DECLARATION_FIELDS: Final[frozenset[str]] = frozenset({"mode", "supported_endpoints"}) - CHAT_SHAPED_MODES: Final[frozenset[str]] = frozenset({"chat", "completion", "responses"}) @@ -411,32 +410,18 @@ _FALLBACK_LIST_ADAPTER: Final[TypeAdapter[list[object]]] = TypeAdapter( # mutab ) -class _LiveCostMap(Protocol): - @property - def model_cost(self) -> Mapping[str, object]: ... - - class _RouterFallbacks(Protocol): @property def fallbacks(self) -> Sequence[object] | None: ... -def _live_cost_map_entry(catalog: _LiveCostMap, catalog_key: str) -> Mapping[str, object]: - try: - return JSON_OBJECT_ADAPTER.validate_python(catalog.model_cost.get(catalog_key) or {}) - except ValidationError: - return MappingProxyType({}) - - def _echoed_from_cost_map(model_info: Mapping[str, object]) -> frozenset[str]: catalog_key: Final = model_info.get(COST_MAP_LOOKUP_KEY) if not isinstance(catalog_key, str): return frozenset() return frozenset( echoed_cost_map_fields( - model_info, - _live_cost_map_entry(litellm, catalog_key), - GetModelCostMap.loaded_model_cost_map().get(catalog_key) or MappingProxyType({}), + model_info, GetModelCostMap.loaded_model_cost_map().get(catalog_key) or MappingProxyType({}) ) ) @@ -447,13 +432,21 @@ def _declared_non_chat_endpoints(deployment: Mapping[str, object]) -> frozenset[ declared: Final = _DeploymentEndpointDeclaration.model_validate(model_info) except ValidationError: return None - if not DECLARATION_FIELDS.isdisjoint(_echoed_from_cost_map(model_info)): - return None mode: Final = (declared.mode or "").strip().lower() if not mode or mode in CHAT_SHAPED_MODES or declared.supported_endpoints is None: return None endpoints: Final = frozenset(declared.supported_endpoints) - return endpoints if endpoints.isdisjoint(CHAT_SHAPED_ENDPOINTS) else None + if not endpoints.isdisjoint(CHAT_SHAPED_ENDPOINTS): + return None + return None if "supported_endpoints" in _echoed_from_cost_map(model_info) else endpoints + + +def _group_deployments(model: str, llm_router: LitellmRouter, team_id: str | None) -> tuple[Mapping[str, object], ...]: + return tuple(llm_router.get_model_list(model_name=model, team_id=team_id) or ()) + + +def _any_serves_chat(deployments: Sequence[Mapping[str, object]]) -> bool: + return any(_declared_non_chat_endpoints(deployment) is None for deployment in deployments) def _override_fallbacks(data: Mapping[str, object]) -> object: @@ -474,22 +467,65 @@ def _router_fallbacks(llm_router: _RouterFallbacks) -> Sequence[object] | None: return llm_router.fallbacks -def _fallback_would_answer(model: str, data: Mapping[str, object], llm_router: LitellmRouter) -> bool: +def _fallback_targets( + model: str, + data: Mapping[str, object], + fallbacks: list[object], # mutable-ok: mirrors get_fallback_model_group's contract +) -> Sequence[object]: + if check_non_standard_fallback_format(fallbacks=fallbacks): + return fallbacks + resolved, _ = get_fallback_model_group_for_lookup_groups( + fallbacks=fallbacks, + lookup_groups=fallback_lookup_groups(data, model), + ) + return resolved or () + + +def _fallback_target_group(target: object) -> str | None: + if isinstance(target, str): + return target + try: + entry: Final = JSON_OBJECT_ADAPTER.validate_python(target) + except ValidationError: + return None + named_model: Final = entry.get("model") + return named_model if isinstance(named_model, str) else None + + +def _fallback_target_answers( + group: str | None, + data: Mapping[str, object], + llm_router: LitellmRouter, + team_id: str | None, + walked: frozenset[str], +) -> bool: + if group is None or group in walked: + return False + deployments: Final = _group_deployments(group, llm_router, team_id) + if not deployments: + return False + return _any_serves_chat(deployments) or _fallback_would_answer(group, data, llm_router, team_id, walked) + + +def _fallback_would_answer( + model: str, + data: Mapping[str, object], + llm_router: LitellmRouter, + team_id: str | None, + walked: frozenset[str] = frozenset(), +) -> bool: if fallbacks_disabled_for_request(data): return False try: fallbacks: Final = _FALLBACK_LIST_ADAPTER.validate_python(_fallbacks_for_request(data, llm_router)) except ValidationError: return False - if not fallbacks: - return False - if check_non_standard_fallback_format(fallbacks=fallbacks): - return True - resolved, _ = get_fallback_model_group_for_lookup_groups( - fallbacks=fallbacks, - lookup_groups=fallback_lookup_groups(data, model), + walked_here: Final = walked | {model} + return any( + has_unattempted_fallback_target((target,), data) + and _fallback_target_answers(_fallback_target_group(target), data, llm_router, team_id, walked_here) + for target in _fallback_targets(model, data, fallbacks) ) - return has_unattempted_fallback_target(resolved, data) def endpoint_unsupported_by_model( @@ -502,15 +538,16 @@ def endpoint_unsupported_by_model( model: Final = data.get("model") if endpoint is None or not isinstance(model, str) or not model or llm_router is None: return None - deployments: Final = tuple(llm_router.get_model_list(model_name=model, team_id=team_id) or ()) - non_chat: Final = tuple( - endpoints for deployment in deployments if (endpoints := _declared_non_chat_endpoints(deployment)) is not None - ) - if not deployments or len(non_chat) < len(deployments) or _fallback_would_answer(model, data, llm_router): + deployments: Final = _group_deployments(model, llm_router, team_id) + if not deployments or _any_serves_chat(deployments): return None - supported: Final = ", ".join(sorted(frozenset[str]().union(*non_chat))) or "none" + if _fallback_would_answer(model, data, llm_router, team_id): + return None + supported: Final = ", ".join( + sorted(frozenset[str]().union(*(_declared_non_chat_endpoints(d) or frozenset() for d in deployments))) + ) return ProxyUnsupportedEndpointError( - message=f"{endpoint} is not supported by model={model}. Supported endpoints: {supported}", + message=f"{endpoint} is not supported by model={model}. Supported endpoints: {supported or 'none'}", type="invalid_request_error", param="model", code=status.HTTP_400_BAD_REQUEST, diff --git a/tests/integration/routing/test_non_chat_model_endpoint_gate.py b/tests/integration/routing/test_non_chat_model_endpoint_gate.py index b12b42ec638..cd78722af9e 100644 --- a/tests/integration/routing/test_non_chat_model_endpoint_gate.py +++ b/tests/integration/routing/test_non_chat_model_endpoint_gate.py @@ -613,6 +613,16 @@ def test_an_empty_request_fallback_list_keeps_the_rejection(rig: _Rig) -> None: _assert_rejected(rig, response, marker, _openai_error("/v1/chat/completions", model, "/v1/embeddings")) +def test_fallbacks_that_cannot_answer_chat_keep_the_rejection(rig: _Rig) -> None: + model: Final = rig.names["embedding_only"] + marker: Final = _marker() + fallbacks: Final = [rig.names["image_generation"], model, f"gate-missing-{uuid.uuid4().hex}"] + response: Final = rig.gateway.request( + "POST", "/v1/chat/completions", {**_chat_body(model, marker), "fallbacks": fallbacks} + ) + _assert_rejected(rig, response, marker, _openai_error("/v1/chat/completions", model, "/v1/embeddings")) + + def test_request_fallbacks_do_not_rescue_an_unknown_primary(rig: _Rig) -> None: marker: Final = _marker() response: Final = rig.gateway.request( diff --git a/tests/unit/proxy/response_api_endpoints/test_endpoints.py b/tests/unit/proxy/response_api_endpoints/test_endpoints.py index 4913d855d83..4f71fedd174 100644 --- a/tests/unit/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/unit/proxy/response_api_endpoints/test_endpoints.py @@ -1015,12 +1015,13 @@ class TestResponsesWSDeclaredNonChatModel: ({"mode": "embedding"}, False), ], ) - async def test_endpoint_rejects_a_model_declared_non_chat_before_routing( + async def test_session_closes_on_a_model_declared_non_chat_before_routing( self, query_model: str | None, model_info: dict[str, object], rejected: bool ): from litellm.proxy.response_api_endpoints.endpoints import ( responses_websocket_endpoint, ) + from litellm.proxy.route_llm_request import endpoint_unsupported_by_model router: Final = litellm.Router( model_list=[ @@ -1043,10 +1044,16 @@ class TestResponsesWSDeclaredNonChatModel: ws.send_text = AsyncMock() ws.close = AsyncMock() + async def gated_pre_call(**kwargs): + unsupported = endpoint_unsupported_by_model( + route_type=kwargs["route_type"], data={"model": kwargs["model"]}, llm_router=router, team_id=None + ) + if unsupported is not None: + raise unsupported + return {"model": kwargs["model"], "litellm_metadata": {}}, MagicMock() + processor = MagicMock() - processor.common_processing_pre_call_logic = AsyncMock( - return_value=({"model": "embed-only", "litellm_metadata": {}}, MagicMock()) - ) + processor.common_processing_pre_call_logic = AsyncMock(side_effect=gated_pre_call) async def fake_llm_call(): return None @@ -1084,7 +1091,6 @@ class TestResponsesWSDeclaredNonChatModel: "/v1/responses is not supported by model=embed-only. Supported endpoints: /v1/embeddings" ) ws.close.assert_awaited_once_with(code=1008, reason="Pre-call error") - processor.common_processing_pre_call_logic.assert_not_awaited() mock_route_request.assert_not_awaited() diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index 3a849dc0a9b..bfee48f8b7b 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -725,12 +725,20 @@ class TestProxyBaseLLMRequestProcessing: ) @pytest.mark.asyncio + @pytest.mark.parametrize( + "route_type, endpoint", + [("acompletion", "/v1/chat/completions"), ("_aresponses_websocket", "/v1/responses")], + ) + @pytest.mark.parametrize( + "requested_model, key_aliases", + [("embed-only", {}), ("embed-alias", {"embed-alias": "embed-only"})], + ) async def test_common_processing_pre_call_logic_rejects_chat_on_embedding_only_model_before_pre_call_hooks( - self, monkeypatch + self, monkeypatch, route_type, endpoint, requested_model, key_aliases ): processing_obj, mock_request, mock_proxy_logging_obj, mock_proxy_config, _ = self._guardrail_tag_budget_harness( monkeypatch, - request_body={"model": "embed-only", "messages": [{"role": "user", "content": "hello"}]}, + request_body={"model": requested_model, "messages": [{"role": "user", "content": "hello"}]}, guardrail_tags=[], ) @@ -738,16 +746,16 @@ class TestProxyBaseLLMRequestProcessing: await processing_obj.common_processing_pre_call_logic( request=mock_request, general_settings={}, - user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-test"), + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-test", aliases=key_aliases), proxy_logging_obj=mock_proxy_logging_obj, proxy_config=mock_proxy_config, - route_type="acompletion", + route_type=route_type, llm_router=self._router_with_embedding_only_model(), ) assert exc_info.value.code == "400" assert exc_info.value.message == ( - "/v1/chat/completions is not supported by model=embed-only. Supported endpoints: /v1/embeddings" + f"{endpoint} is not supported by model=embed-only. Supported endpoints: /v1/embeddings" ) mock_proxy_logging_obj.pre_call_hook.assert_not_awaited() diff --git a/tests/unit/proxy/test_route_llm_request.py b/tests/unit/proxy/test_route_llm_request.py index 87dc9aee7e2..475aeaa896d 100644 --- a/tests/unit/proxy/test_route_llm_request.py +++ b/tests/unit/proxy/test_route_llm_request.py @@ -3,13 +3,16 @@ import pytest from types import SimpleNamespace from typing import Final -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock from fastapi import HTTPException import litellm from litellm import Router +from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.proxy.route_llm_request import ( + CHAT_SHAPED_ENDPOINTS, + CHAT_SHAPED_MODES, ProxyModelNotFoundError, ProxyUnsupportedEndpointError, endpoint_unsupported_by_model, @@ -1777,6 +1780,7 @@ CHAT_SHAPED_ROUTES: Final = ( ("agenerate_content", "generateContent"), ("agenerate_content_stream", "streamGenerateContent"), ("acreate_interaction", "/v1beta/interactions"), + ("_aresponses_websocket", "/v1/responses"), ) CHAT_SHAPED_ROUTE_TYPES: Final = tuple(route_type for route_type, _ in CHAT_SHAPED_ROUTES) @@ -1817,7 +1821,7 @@ def test_endpoint_unsupported_rejects_chat_routes_on_embedding_only_model(route_ assert error.message == f"{endpoint} is not supported by model=embed-only. Supported endpoints: /v1/embeddings" -@pytest.mark.parametrize("route_type", ["aembedding", "aimage_generation", "_aresponses_websocket"]) +@pytest.mark.parametrize("route_type", ["aembedding", "aimage_generation", "aspeech", "_arealtime"]) def test_endpoint_unsupported_leaves_non_chat_routes_alone(route_type): router: Final = _router_with(_embedding_deployment("embed-only", EMBEDDING_ONLY)) @@ -1944,6 +1948,8 @@ CHAT_FALLBACK_DEPLOYMENT: Final = { "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "test-key"}, } +EMBEDDING_FALLBACK_DEPLOYMENT: Final = _embedding_deployment("embed-fallback", EMBEDDING_ONLY) + @pytest.mark.parametrize( "router_kwargs, request_kwargs", @@ -1953,11 +1959,18 @@ CHAT_FALLBACK_DEPLOYMENT: Final = { ({}, {"fallbacks": ["chat-fallback"]}), ({}, {"fallbacks": [{"embed-only": ["chat-fallback"]}]}), ({}, {"router_settings_override": {"fallbacks": [{"embed-only": ["chat-fallback"]}]}}), + ({"fallbacks": [{"embed-only": ["embed-fallback", "missing-group", "chat-fallback"]}]}, {}), + ({}, {"fallbacks": [{"model": "chat-fallback", "temperature": 0.2}]}), + ({"fallbacks": [{"embed-only": ["chat-alias"]}], "model_group_alias": {"chat-alias": "chat-fallback"}}, {}), + ({"fallbacks": [{"embed-only": ["embed-fallback"]}, {"embed-fallback": ["chat-fallback"]}]}, {}), ], ) def test_endpoint_unsupported_lets_a_configured_fallback_answer(router_kwargs, request_kwargs): router: Final = _router_with( - _embedding_deployment("embed-only", EMBEDDING_ONLY), CHAT_FALLBACK_DEPLOYMENT, **router_kwargs + _embedding_deployment("embed-only", EMBEDDING_ONLY), + CHAT_FALLBACK_DEPLOYMENT, + EMBEDDING_FALLBACK_DEPLOYMENT, + **router_kwargs, ) assert _unsupported("acompletion", "embed-only", router, **request_kwargs) is None @@ -1974,35 +1987,75 @@ def test_endpoint_unsupported_lets_a_configured_fallback_answer(router_kwargs, r {"router_settings_override": {"fallbacks": [{"other-group": ["chat-fallback"]}]}}, ), ({}, {"fallbacks": [], "router_settings_override": {"fallbacks": [{"embed-only": ["chat-fallback"]}]}}), + ({"fallbacks": [{"embed-only": ["embed-fallback"]}]}, {}), + ({"fallbacks": [{"embed-only": ["embed-only"]}]}, {}), + ({"fallbacks": [{"embed-only": ["missing-group"]}]}, {}), + ({"default_fallbacks": ["embed-fallback"]}, {}), + ({}, {"fallbacks": ["embed-fallback", "missing-group"]}), + ({}, {"fallbacks": [{"model": "embed-fallback", "temperature": 0.2}]}), + ({"fallbacks": [{"embed-only": ["embed-alias"]}], "model_group_alias": {"embed-alias": "embed-fallback"}}, {}), + ({"fallbacks": [{"embed-only": ["embed-fallback"]}, {"embed-fallback": ["embed-only"]}]}, {}), + ({"fallbacks": [{"embed-only": ["embed-fallback"]}, {"embed-fallback": ["chat-fallback"]}]}, {"fallbacks": []}), ], ) def test_endpoint_unsupported_rejects_when_no_fallback_would_answer(router_kwargs, request_kwargs): router: Final = _router_with( - _embedding_deployment("embed-only", EMBEDDING_ONLY), CHAT_FALLBACK_DEPLOYMENT, **router_kwargs + _embedding_deployment("embed-only", EMBEDDING_ONLY), + CHAT_FALLBACK_DEPLOYMENT, + EMBEDDING_FALLBACK_DEPLOYMENT, + **router_kwargs, ) - assert isinstance(_unsupported("acompletion", "embed-only", router, **request_kwargs), ProxyUnsupportedEndpointError) + assert isinstance( + _unsupported("acompletion", "embed-only", router, **request_kwargs), ProxyUnsupportedEndpointError + ) -ECHOED_CATALOG_ENTRY: Final = {"mode": "image_generation", "supported_endpoints": ["/v1/images/generations"]} +def _catalog_key_declaring_non_chat() -> str: + return min( + key + for key, entry in GetModelCostMap.loaded_model_cost_map().items() + if entry.get("litellm_provider") == "openai" + and entry.get("mode") not in CHAT_SHAPED_MODES + and isinstance(endpoints := entry.get("supported_endpoints"), list) + and CHAT_SHAPED_ENDPOINTS.isdisjoint(endpoints) + ) + + +def _catalog_declaration(catalog_key: str) -> dict[str, object]: + entry: Final = GetModelCostMap.loaded_model_cost_map()[catalog_key] + return {"mode": entry["mode"], "supported_endpoints": entry["supported_endpoints"]} @pytest.mark.parametrize( - "model_info, rejected", + "declaration, with_key, rejected", [ - ({"key": "echoed-image-model", **ECHOED_CATALOG_ENTRY}, False), - ({"key": "echoed-image-model", "mode": "image_generation", "supported_endpoints": ["/v1/embeddings"]}, False), - ({"key": "echoed-image-model", "mode": "embedding", "supported_endpoints": ["/v1/embeddings"]}, True), - (dict(ECHOED_CATALOG_ENTRY), True), + pytest.param("catalog", True, False, id="echo-of-the-catalog-row"), + pytest.param("catalog-mode-typed-endpoints", True, True, id="endpoints-typed-over-an-echo"), + pytest.param("typed-mode-catalog-endpoints", True, False, id="endpoints-left-as-echoed"), + pytest.param("typed", True, True, id="both-typed-over-an-echo"), + pytest.param("catalog", False, True, id="catalog-values-typed-by-hand"), ], ) -def test_endpoint_unsupported_ignores_fields_copied_from_the_cost_map(model_info, rejected): - router: Final = _router_with(_embedding_deployment("image-row", model_info)) +def test_endpoint_unsupported_ignores_endpoints_copied_from_the_loaded_catalog(declaration, with_key, rejected): + catalog_key: Final = _catalog_key_declaring_non_chat() + catalog: Final = _catalog_declaration(catalog_key) + declared: Final = { + "catalog": catalog, + "catalog-mode-typed-endpoints": {**catalog, "supported_endpoints": ["/v1/embeddings"]}, + "typed-mode-catalog-endpoints": {**catalog, "mode": "embedding"}, + "typed": EMBEDDING_ONLY, + }[declaration] + router: Final = _router_with( + { + "model_name": "catalog-row", + "litellm_params": {"model": catalog_key, "api_key": "test-key"}, + "model_info": {**({"key": catalog_key} if with_key else {}), **declared}, + } + ) - with patch.dict(litellm.model_cost, {"echoed-image-model": ECHOED_CATALOG_ENTRY}): - error: Final = _unsupported("acompletion", "image-row", router) - - assert isinstance(error, ProxyUnsupportedEndpointError) is rejected + assert litellm.model_cost[catalog_key]["supported_endpoints"] == declared["supported_endpoints"] + assert isinstance(_unsupported("acompletion", "catalog-row", router), ProxyUnsupportedEndpointError) is rejected def _router_with_defaults(default_litellm_params: dict[str, object]):