diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index 37f292816f7..972eb6dda46 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -381,14 +381,10 @@ CHAT_SHAPED_ENDPOINT_BY_ROUTE_TYPE: Final[Mapping[str, str]] = MappingProxyType( CHAT_SHAPED_ENDPOINTS: Final[frozenset[str]] = frozenset( { - "/v1/chat/completions", "/chat/completions", - "/v1/completions", "/completions", - "/v1/messages", - "/v1/responses", "/responses", - "/v1beta/interactions", + *CHAT_SHAPED_ENDPOINT_BY_ROUTE_TYPE.values(), } ) @@ -455,8 +451,37 @@ def _declared_non_chat_endpoints(deployment: Mapping[str, object]) -> frozenset[ return None if _declaration_overruled_by_catalog(model_info) else endpoints +def _deployment_by_id(model_id: str, llm_router: LitellmRouter) -> tuple[Mapping[str, object], ...]: + try: + return (JSON_OBJECT_ADAPTER.validate_python(llm_router.get_model_info(id=model_id)),) + except ValidationError: + return () + + def _group_deployments(model: str, llm_router: LitellmRouter, team_id: str | None) -> tuple[Mapping[str, object], ...]: - return tuple(llm_router.get_model_list(model_name=model, team_id=team_id) or ()) + return tuple(llm_router.get_model_list(model_name=model, team_id=team_id) or ()) or _deployment_by_id( + model, llm_router + ) + + +def _deployments_by_backend_name(model: str, llm_router: LitellmRouter) -> tuple[Mapping[str, object], ...]: + if ( + model not in llm_router.deployment_names + or llm_router.router_general_settings.pass_through_all_models + or llm_router.pattern_router.patterns + ): + return () + return tuple( + deployment + for deployment in llm_router.get_model_list() or () + if deployment["litellm_params"].get("model") == model + ) + + +def _requested_deployments( + model: str, llm_router: LitellmRouter, team_id: str | None +) -> tuple[Mapping[str, object], ...]: + return _group_deployments(model, llm_router, team_id) or _deployments_by_backend_name(model, llm_router) def _any_serves_chat(deployments: Sequence[Mapping[str, object]]) -> bool: @@ -587,7 +612,7 @@ def endpoint_unsupported_by_model( model: Final = data.get("model") if endpoint is None or not isinstance(model, str) or not model or llm_router is None: return None - deployments: Final = _group_deployments(model, llm_router, team_id) + deployments: Final = _requested_deployments(model, llm_router, team_id) if not deployments or _any_serves_chat(deployments): return None if _fallback_would_answer(model, data, llm_router, team_id): diff --git a/tests/unit/proxy/test_route_llm_request.py b/tests/unit/proxy/test_route_llm_request.py index 5e4b5452bfd..709b4569bbe 100644 --- a/tests/unit/proxy/test_route_llm_request.py +++ b/tests/unit/proxy/test_route_llm_request.py @@ -10,6 +10,7 @@ from fastapi import HTTPException import litellm from litellm import Router from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap +from litellm.types.router import RouterGeneralSettings from litellm.proxy.route_llm_request import ( CHAT_SHAPED_ENDPOINTS, CHAT_SHAPED_MODES, @@ -1877,6 +1878,9 @@ def test_endpoint_unsupported_needs_a_non_chat_mode_to_reject(route_type, model_ ["/v1/completions"], ["/completions"], ["/v1beta/interactions"], + ["generateContent"], + ["streamGenerateContent"], + ["/v1/responses/compact"], ], ) def test_endpoint_unsupported_allows_every_chat_route_when_any_chat_endpoint_is_declared(route_type, declared): @@ -1933,6 +1937,35 @@ def test_endpoint_unsupported_follows_a_model_group_alias(): assert error.message == "/v1/messages is not supported by model=embed-alias. Supported endpoints: /v1/embeddings" +@pytest.mark.parametrize("model", ["embed-1", "sagemaker/openai/embed-endpoint"]) +def test_endpoint_unsupported_rejects_chat_sent_to_the_deployment_id_or_backend_model(model): + router: Final = _router_with( + _embedding_deployment("embed-only", {**EMBEDDING_ONLY, "id": "embed-1"}), CHAT_FALLBACK_DEPLOYMENT + ) + + error: Final = _unsupported("acompletion", model, router) + + assert isinstance(error, ProxyUnsupportedEndpointError) + assert error.message == ( + f"/v1/chat/completions is not supported by model={model}. Supported endpoints: /v1/embeddings" + ) + + +@pytest.mark.parametrize( + "extra_deployments, router_kwargs", + [ + ((), {"router_general_settings": RouterGeneralSettings(pass_through_all_models=True)}), + (({"model_name": "openai/*", "litellm_params": {"model": "openai/*", "api_key": "test-key"}},), {}), + ], +) +def test_endpoint_unsupported_leaves_a_backend_model_to_the_route_that_serves_it(extra_deployments, router_kwargs): + router: Final = _router_with( + _embedding_deployment("embed-only", EMBEDDING_ONLY), *extra_deployments, **router_kwargs + ) + + assert _unsupported("acompletion", "sagemaker/openai/embed-endpoint", router) is None + + @pytest.mark.parametrize("model", ["unknown-model", "", None]) def test_endpoint_unsupported_ignores_models_without_deployments(model): router: Final = _router_with(_embedding_deployment("embed-only", EMBEDDING_ONLY)) @@ -1947,9 +1980,12 @@ def test_endpoint_unsupported_skips_the_check_without_a_router(): CHAT_FALLBACK_DEPLOYMENT: Final = { "model_name": "chat-fallback", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "test-key"}, + "model_info": {"id": "chat-fallback-id"}, } -EMBEDDING_FALLBACK_DEPLOYMENT: Final = _embedding_deployment("embed-fallback", EMBEDDING_ONLY) +EMBEDDING_FALLBACK_DEPLOYMENT: Final = _embedding_deployment( + "embed-fallback", {**EMBEDDING_ONLY, "id": "embed-fallback-id"} +) @pytest.mark.parametrize( @@ -1966,6 +2002,7 @@ EMBEDDING_FALLBACK_DEPLOYMENT: Final = _embedding_deployment("embed-fallback", E ({"fallbacks": [{"embed-only": ["embed-fallback"]}, {"embed-fallback": ["chat-fallback"]}]}, {}), ({"fallbacks": [{"embed-only": ["chat-fallback"]}]}, {"metadata": {"_disable_fallbacks": True}}), ({}, {"fallbacks": {"chat-fallback": ["embed-fallback"]}}), + ({}, {"fallbacks": ["chat-fallback-id"]}), ], ) def test_endpoint_unsupported_lets_a_configured_fallback_answer(router_kwargs, request_kwargs): @@ -1995,6 +2032,7 @@ def test_endpoint_unsupported_lets_a_configured_fallback_answer(router_kwargs, r ({"fallbacks": [{"embed-only": ["missing-group"]}]}, {}), ({"default_fallbacks": ["embed-fallback"]}, {}), ({}, {"fallbacks": ["embed-fallback", "missing-group"]}), + ({}, {"fallbacks": ["embed-fallback-id"]}), ({}, {"fallbacks": [{"model": "embed-fallback", "temperature": 0.2}]}), ({"fallbacks": [{"embed-only": ["embed-alias"]}], "model_group_alias": {"embed-alias": "embed-fallback"}}, {}), ({"fallbacks": [{"embed-only": ["embed-fallback"]}, {"embed-fallback": ["embed-only"]}]}, {}),