diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index b274121822e..37f292816f7 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -416,14 +416,27 @@ class _RouterFallbacks(Protocol): def fallbacks(self) -> Sequence[object] | None: ... -def _echoed_from_cost_map(model_info: Mapping[str, object]) -> frozenset[str]: +def _loaded_catalog_entry(model_info: Mapping[str, object]) -> Mapping[str, object]: catalog_key: Final = model_info.get(COST_MAP_LOOKUP_KEY) if not isinstance(catalog_key, str): - return frozenset() - return frozenset( - echoed_cost_map_fields( - model_info, GetModelCostMap.loaded_model_cost_map().get(catalog_key) or MappingProxyType({}) - ) + return MappingProxyType({}) + return GetModelCostMap.loaded_model_cost_map().get(catalog_key) or MappingProxyType({}) + + +def _catalog_serves_chat(catalog_entry: Mapping[str, object]) -> bool: + try: + declared: Final = _DeploymentEndpointDeclaration.model_validate(dict(catalog_entry)) + except ValidationError: + return False + return (declared.mode or "").strip().lower() in CHAT_SHAPED_MODES or not CHAT_SHAPED_ENDPOINTS.isdisjoint( + declared.supported_endpoints or () + ) + + +def _declaration_overruled_by_catalog(model_info: Mapping[str, object]) -> bool: + catalog_entry: Final = _loaded_catalog_entry(model_info) + return _catalog_serves_chat(catalog_entry) or "supported_endpoints" in echoed_cost_map_fields( + model_info, catalog_entry ) @@ -439,7 +452,7 @@ def _declared_non_chat_endpoints(deployment: Mapping[str, object]) -> frozenset[ endpoints: Final = frozenset(declared.supported_endpoints) if not endpoints.isdisjoint(CHAT_SHAPED_ENDPOINTS): return None - return None if "supported_endpoints" in _echoed_from_cost_map(model_info) else endpoints + return None if _declaration_overruled_by_catalog(model_info) else endpoints def _group_deployments(model: str, llm_router: LitellmRouter, team_id: str | None) -> tuple[Mapping[str, object], ...]: diff --git a/tests/unit/proxy/test_route_llm_request.py b/tests/unit/proxy/test_route_llm_request.py index c9128eba8c2..5e4b5452bfd 100644 --- a/tests/unit/proxy/test_route_llm_request.py +++ b/tests/unit/proxy/test_route_llm_request.py @@ -2061,6 +2061,45 @@ def test_endpoint_unsupported_ignores_endpoints_copied_from_the_loaded_catalog(d assert isinstance(_unsupported("acompletion", "catalog-row", router), ProxyUnsupportedEndpointError) is rejected +def _catalog_key_serving_chat_by_mode() -> str: + return min( + key + for key, entry in GetModelCostMap.loaded_model_cost_map().items() + if entry.get("mode") == "chat" and "supported_endpoints" not in entry + ) + + +def _catalog_key_serving_chat_by_endpoints() -> str: + return min( + key + for key, entry in GetModelCostMap.loaded_model_cost_map().items() + if entry.get("mode") not in CHAT_SHAPED_MODES + and isinstance(endpoints := entry.get("supported_endpoints"), list) + and not CHAT_SHAPED_ENDPOINTS.isdisjoint(endpoints) + ) + + +@pytest.mark.parametrize( + "catalog_key, rejected", + [ + pytest.param(_catalog_key_serving_chat_by_mode, False, id="key-of-a-chat-mode-catalog-entry"), + pytest.param(_catalog_key_serving_chat_by_endpoints, False, id="key-of-a-catalog-entry-listing-chat"), + pytest.param(_catalog_key_declaring_non_chat, True, id="key-of-a-non-chat-catalog-entry"), + pytest.param(lambda: "not-in-the-catalog", True, id="key-missing-from-the-catalog"), + ], +) +def test_endpoint_unsupported_ignores_non_chat_values_on_a_row_whose_catalog_entry_serves_chat(catalog_key, rejected): + router: Final = _router_with( + { + "model_name": "catalog-row", + "litellm_params": {"model": "openai/catalog-row", "api_key": "test-key"}, + "model_info": {"key": catalog_key(), **EMBEDDING_ONLY}, + } + ) + + assert isinstance(_unsupported("acompletion", "catalog-row", router), ProxyUnsupportedEndpointError) is rejected + + class _LookupCountingRouter(Router): def __init__(self, **router_kwargs: object) -> None: self.looked_up: list[str | None] = []