diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index ae294871afc..f17406c054d 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -340,6 +340,80 @@ def _raise_on_strategy_router_write_violation( ) +def _validate_routable_model_deployment( + incoming_params: GenericLiteLLMParams | None, + existing_params: GenericLiteLLMParams | None = None, +) -> str | None: + """Validate that incoming litellm_params specifies a routable LLM provider before DB insertion. + + When adding or patching a model deployment, the router must be able to resolve its provider. + Unroutable models (such as typesafe/* without a chat completion provider) get dropped by the + router upon reload, causing a 500 degraded serving error and leaving dead zombie rows in + LiteLLM_ProxyModelTable. Pre-validating the provider returns a clean 400 Bad Request instead. + """ + if incoming_params is None: + return None + + model = getattr(incoming_params, "model", None) + if model is None and existing_params is not None: + model = getattr(existing_params, "model", None) + + if not model or not isinstance(model, str): + return None + + # Auto-router and strategy-router pseudo-models are validated separately + if model.startswith("auto_router/") or model.startswith("strategy_router/"): + return None + + custom_llm_provider = getattr(incoming_params, "custom_llm_provider", None) or ( + getattr(existing_params, "custom_llm_provider", None) if existing_params else None + ) + api_base = getattr(incoming_params, "api_base", None) or ( + getattr(existing_params, "api_base", None) if existing_params else None + ) + api_key = getattr(incoming_params, "api_key", None) or ( + getattr(existing_params, "api_key", None) if existing_params else None + ) + + try: + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + get_llm_provider( + model=model, + custom_llm_provider=custom_llm_provider, + api_base=api_base, + api_key=api_key, + litellm_params=incoming_params, + ) + except Exception as e: + err_msg = str(e) + if "typesafe" in model.lower() or (custom_llm_provider and "typesafe" in custom_llm_provider.lower()): + return ( + f"Model '{model}' cannot be registered as a chat completion deployment: 'typesafe' is not a routable chat provider. " + "TypeSafe Jev is supported via pass-through endpoints (/typesafe/*) or as an auto-router complexity classifier " + f"(router_settings.classifier_type: 'jev'). Provider error: {err_msg}" + ) + return f"Model '{model}' is not a routable provider: {err_msg}" + return None + + +def _raise_on_unroutable_model_deployment( + incoming_params: GenericLiteLLMParams | None, + existing_params: GenericLiteLLMParams | None = None, +) -> None: + violation = _validate_routable_model_deployment( + incoming_params=incoming_params, existing_params=existing_params + ) + if violation is None: + return + raise ProxyException( + message=violation, + type=ProxyErrorTypes.bad_request_error.value, + code=status.HTTP_400_BAD_REQUEST, + param="litellm_params.model", + ) + + def _stored_credential_name(existing_litellm_params: GenericLiteLLMParams | None) -> str | None: if existing_litellm_params is None or existing_litellm_params.litellm_credential_name is None: return None @@ -1219,6 +1293,10 @@ async def patch_model( incoming_params=patch_data.litellm_params, existing_params=db_model.litellm_params, ) + _raise_on_unroutable_model_deployment( + incoming_params=patch_data.litellm_params, + existing_params=db_model.litellm_params, + ) effective_params: Final = _effective_complexity_router_params( patch_data.litellm_params, db_model.litellm_params @@ -2436,6 +2514,10 @@ async def add_new_model( incoming_params=model_params.litellm_params, existing_params=None, ) + _raise_on_unroutable_model_deployment( + incoming_params=model_params.litellm_params, + existing_params=None, + ) _raise_if_rate_limits_required_but_missing( litellm_params=model_params.litellm_params, diff --git a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py index 5f7807650e1..c9d236bd454 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py @@ -5986,6 +5986,45 @@ class TestStrategyRouterWriteValidation: is None ) + def test_typesafe_model_rejected_with_helpful_guidance(self): + from litellm.proxy.management_endpoints.model_management_endpoints import ( + _validate_routable_model_deployment, + ) + from litellm.types.router import LiteLLM_Params + + params = LiteLLM_Params( + model="typesafe/jev-latest", + api_key="sk-test", + ) + violation = _validate_routable_model_deployment(params) + assert violation is not None + assert "typesafe" in violation.lower() + assert "not a routable chat provider" in violation or "pass-through" in violation + + def test_routable_model_accepted(self): + from litellm.proxy.management_endpoints.model_management_endpoints import ( + _validate_routable_model_deployment, + ) + from litellm.types.router import LiteLLM_Params + + params = LiteLLM_Params( + model="openai/gpt-4o-mini", + api_key="sk-test", + ) + assert _validate_routable_model_deployment(params) is None + + def test_auto_router_model_skipped(self): + from litellm.proxy.management_endpoints.model_management_endpoints import ( + _validate_routable_model_deployment, + ) + from litellm.types.router import LiteLLM_Params + + params = LiteLLM_Params( + model="auto_router/complexity_router", + api_key="sk-test", + ) + assert _validate_routable_model_deployment(params) is None + @staticmethod def _live_router_holding_one_capability(limit: int | None, config: Mapping[str, object]) -> Router: return Router(