fix(proxy): check chat sent to a deployment id or backend model name

litellm/proxy/route_llm_request.py

@@ -381,14 +381,10 @@ CHAT_SHAPED_ENDPOINT_BY_ROUTE_TYPE: Final[Mapping[str, str]] = MappingProxyType(
 
 CHAT_SHAPED_ENDPOINTS: Final[frozenset[str]] = frozenset(
     {
-        "/v1/chat/completions",
         "/chat/completions",
-        "/v1/completions",
         "/completions",
-        "/v1/messages",
-        "/v1/responses",
         "/responses",
-        "/v1beta/interactions",
+        *CHAT_SHAPED_ENDPOINT_BY_ROUTE_TYPE.values(),
     }
 )
 
@@ -455,8 +451,37 @@ def _declared_non_chat_endpoints(deployment: Mapping[str, object]) -> frozenset[
     return None if _declaration_overruled_by_catalog(model_info) else endpoints
 
 
+def _deployment_by_id(model_id: str, llm_router: LitellmRouter) -> tuple[Mapping[str, object], ...]:
+    try:
+        return (JSON_OBJECT_ADAPTER.validate_python(llm_router.get_model_info(id=model_id)),)
+    except ValidationError:
+        return ()
+
+
 def _group_deployments(model: str, llm_router: LitellmRouter, team_id: str | None) -> tuple[Mapping[str, object], ...]:
-    return tuple(llm_router.get_model_list(model_name=model, team_id=team_id) or ())
+    return tuple(llm_router.get_model_list(model_name=model, team_id=team_id) or ()) or _deployment_by_id(
+        model, llm_router
+    )
+
+
+def _deployments_by_backend_name(model: str, llm_router: LitellmRouter) -> tuple[Mapping[str, object], ...]:
+    if (
+        model not in llm_router.deployment_names
+        or llm_router.router_general_settings.pass_through_all_models
+        or llm_router.pattern_router.patterns
+    ):
+        return ()
+    return tuple(
+        deployment
+        for deployment in llm_router.get_model_list() or ()
+        if deployment["litellm_params"].get("model") == model
+    )
+
+
+def _requested_deployments(
+    model: str, llm_router: LitellmRouter, team_id: str | None
+) -> tuple[Mapping[str, object], ...]:
+    return _group_deployments(model, llm_router, team_id) or _deployments_by_backend_name(model, llm_router)
 
 
 def _any_serves_chat(deployments: Sequence[Mapping[str, object]]) -> bool:
@@ -587,7 +612,7 @@ def endpoint_unsupported_by_model(
     model: Final = data.get("model")
     if endpoint is None or not isinstance(model, str) or not model or llm_router is None:
         return None
-    deployments: Final = _group_deployments(model, llm_router, team_id)
+    deployments: Final = _requested_deployments(model, llm_router, team_id)
     if not deployments or _any_serves_chat(deployments):
         return None
     if _fallback_would_answer(model, data, llm_router, team_id):

tests/unit/proxy/test_route_llm_request.py

@@ -10,6 +10,7 @@ from fastapi import HTTPException
 import litellm
 from litellm import Router
 from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
+from litellm.types.router import RouterGeneralSettings
 from litellm.proxy.route_llm_request import (
     CHAT_SHAPED_ENDPOINTS,
     CHAT_SHAPED_MODES,
@@ -1877,6 +1878,9 @@ def test_endpoint_unsupported_needs_a_non_chat_mode_to_reject(route_type, model_
         ["/v1/completions"],
         ["/completions"],
         ["/v1beta/interactions"],
+        ["generateContent"],
+        ["streamGenerateContent"],
+        ["/v1/responses/compact"],
     ],
 )
 def test_endpoint_unsupported_allows_every_chat_route_when_any_chat_endpoint_is_declared(route_type, declared):
@@ -1933,6 +1937,35 @@ def test_endpoint_unsupported_follows_a_model_group_alias():
     assert error.message == "/v1/messages is not supported by model=embed-alias. Supported endpoints: /v1/embeddings"
 
 
+@pytest.mark.parametrize("model", ["embed-1", "sagemaker/openai/embed-endpoint"])
+def test_endpoint_unsupported_rejects_chat_sent_to_the_deployment_id_or_backend_model(model):
+    router: Final = _router_with(
+        _embedding_deployment("embed-only", {**EMBEDDING_ONLY, "id": "embed-1"}), CHAT_FALLBACK_DEPLOYMENT
+    )
+
+    error: Final = _unsupported("acompletion", model, router)
+
+    assert isinstance(error, ProxyUnsupportedEndpointError)
+    assert error.message == (
+        f"/v1/chat/completions is not supported by model={model}. Supported endpoints: /v1/embeddings"
+    )
+
+
+@pytest.mark.parametrize(
+    "extra_deployments, router_kwargs",
+    [
+        ((), {"router_general_settings": RouterGeneralSettings(pass_through_all_models=True)}),
+        (({"model_name": "openai/*", "litellm_params": {"model": "openai/*", "api_key": "test-key"}},), {}),
+    ],
+)
+def test_endpoint_unsupported_leaves_a_backend_model_to_the_route_that_serves_it(extra_deployments, router_kwargs):
+    router: Final = _router_with(
+        _embedding_deployment("embed-only", EMBEDDING_ONLY), *extra_deployments, **router_kwargs
+    )
+
+    assert _unsupported("acompletion", "sagemaker/openai/embed-endpoint", router) is None
+
+
 @pytest.mark.parametrize("model", ["unknown-model", "", None])
 def test_endpoint_unsupported_ignores_models_without_deployments(model):
     router: Final = _router_with(_embedding_deployment("embed-only", EMBEDDING_ONLY))
@@ -1947,9 +1980,12 @@ def test_endpoint_unsupported_skips_the_check_without_a_router():
 CHAT_FALLBACK_DEPLOYMENT: Final = {
     "model_name": "chat-fallback",
     "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "test-key"},
+    "model_info": {"id": "chat-fallback-id"},
 }
 
-EMBEDDING_FALLBACK_DEPLOYMENT: Final = _embedding_deployment("embed-fallback", EMBEDDING_ONLY)
+EMBEDDING_FALLBACK_DEPLOYMENT: Final = _embedding_deployment(
+    "embed-fallback", {**EMBEDDING_ONLY, "id": "embed-fallback-id"}
+)
 
 
 @pytest.mark.parametrize(
@@ -1966,6 +2002,7 @@ EMBEDDING_FALLBACK_DEPLOYMENT: Final = _embedding_deployment("embed-fallback", E
         ({"fallbacks": [{"embed-only": ["embed-fallback"]}, {"embed-fallback": ["chat-fallback"]}]}, {}),
         ({"fallbacks": [{"embed-only": ["chat-fallback"]}]}, {"metadata": {"_disable_fallbacks": True}}),
         ({}, {"fallbacks": {"chat-fallback": ["embed-fallback"]}}),
+        ({}, {"fallbacks": ["chat-fallback-id"]}),
     ],
 )
 def test_endpoint_unsupported_lets_a_configured_fallback_answer(router_kwargs, request_kwargs):
@@ -1995,6 +2032,7 @@ def test_endpoint_unsupported_lets_a_configured_fallback_answer(router_kwargs, r
         ({"fallbacks": [{"embed-only": ["missing-group"]}]}, {}),
         ({"default_fallbacks": ["embed-fallback"]}, {}),
         ({}, {"fallbacks": ["embed-fallback", "missing-group"]}),
+        ({}, {"fallbacks": ["embed-fallback-id"]}),
         ({}, {"fallbacks": [{"model": "embed-fallback", "temperature": 0.2}]}),
         ({"fallbacks": [{"embed-only": ["embed-alias"]}], "model_group_alias": {"embed-alias": "embed-fallback"}}, {}),
         ({"fallbacks": [{"embed-only": ["embed-fallback"]}, {"embed-fallback": ["embed-only"]}]}, {}),