From edb2961e7d12b19ff5c8e3f66552eeab4ff009e1 Mon Sep 17 00:00:00 2001 From: TheRWX <166471970+TheRWX@users.noreply.github.com> Date: Thu, 17 Sep 2026 21:20:20 -0600 Subject: [PATCH] fix(proxy): reject unroutable and pricing-only TypeSafe models before DB write (#41714) - Validate model routability in model management write endpoints (add_new_model, patch_model, update_model) prior to DB persistence - Reject typesafe/* models with HTTP 400 explaining that TypeSafe models are pricing-only, directing users to /typesafe/{endpoint} pass-through and classifier_type: jev - Reject unrecognized providers before persisting rows to LiteLLM_ProxyModelTable, preventing corrupted rows that fail router reload and cause 500 degraded serving errors - Add comprehensive hermetic unit tests, positive controls, duck-typing, runtime annotation contracts, and Hypothesis property-based fuzzing Fixes #41714 --- .../model_management_endpoints.py | 82 ++++++ .../test_model_management_endpoints.py | 258 ++++++++++++++++++ 2 files changed, 340 insertions(+) diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index ffa58d71da8..d0fc51c46bd 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -328,6 +328,84 @@ def _raise_on_strategy_router_write_violation( ) +def _raise_if_unroutable_model(litellm_params: GenericLiteLLMParams | None) -> None: + """Validate that the deployment specifies a routable model and provider before persisting to DB. + + Rejects models that are pricing-only (such as 'typesafe/*') or whose provider cannot be resolved + by the router, preventing corrupted rows from being committed to LiteLLM_ProxyModelTable that + later fail router reload and trigger 500 degraded serving errors. + """ + if litellm_params is None: + return + + raw_model = ( + getattr(litellm_params, "model", None) if not isinstance(litellm_params, dict) else litellm_params.get("model") + ) + if not raw_model or not isinstance(raw_model, str): + return + + custom_llm_provider = ( + getattr(litellm_params, "custom_llm_provider", None) + if not isinstance(litellm_params, dict) + else litellm_params.get("custom_llm_provider") + ) + + # 1. TypeSafe models in LiteLLM are pricing-only for passthrough / auto-router classifier + if raw_model.startswith("typesafe/") or custom_llm_provider == "typesafe": + raise ProxyException( + message=( + f"Model '{raw_model}' uses provider 'typesafe'. TypeSafe models in LiteLLM are pricing-only " + "and cannot be added as a routable model deployment. Use the '/typesafe/{endpoint}' pass-through " + "route (https://docs.litellm.ai/docs/pass_through/typesafe) or configure 'classifier_type: jev' " + "in your auto-router settings." + ), + type=ProxyErrorTypes.validation_error.value, + code=status.HTTP_400_BAD_REQUEST, + param="litellm_params.model", + ) + + # 2. Skip prompt-management callbacks, wildcards, and special router routes + if "/" in raw_model: + split_prefix = raw_model.split("/")[0] + if split_prefix in getattr(litellm, "_known_custom_logger_compatible_callbacks", ()): + return + + if raw_model.startswith("*"): + return + + # 3. Validate provider resolution through get_llm_provider + try: + api_base = ( + getattr(litellm_params, "api_base", None) + if not isinstance(litellm_params, dict) + else litellm_params.get("api_base") + ) + _, resolved_provider, _, _ = litellm.get_llm_provider( + model=raw_model, + custom_llm_provider=custom_llm_provider, + api_base=api_base, + ) + from litellm.router import JSONProviderRegistry, is_registered_custom_provider + + if ( + resolved_provider not in litellm.provider_list + and not JSONProviderRegistry.exists(resolved_provider) + and not is_registered_custom_provider(resolved_provider) + ): + raise ValueError( + f"Provider '{resolved_provider}' is not supported. Supported providers: {litellm.provider_list}" + ) + except Exception as e: + if isinstance(e, ProxyException): + raise e + raise ProxyException( + message=f"Invalid or unroutable model '{raw_model}': {e}", + type=ProxyErrorTypes.validation_error.value, + code=status.HTTP_400_BAD_REQUEST, + param="litellm_params.model", + ) + + AUTO_ROUTER_CAPABILITY_SLOT_LOCK_KEY: Final = 5_872_301 _CAPABILITY_LOCK_SQL: Final = "SELECT 1 AS locked FROM pg_advisory_xact_lock($1)" _STORED_LITELLM_PARAMS_SQL: Final = ( @@ -1087,6 +1165,7 @@ async def patch_model( incoming_params=patch_data.litellm_params, existing_params=db_model.litellm_params, ) + _raise_if_unroutable_model(patch_data.litellm_params) effective_params: Final = _effective_complexity_router_params( patch_data.litellm_params, db_model.litellm_params @@ -2306,6 +2385,8 @@ async def add_new_model( enforced=bool(general_settings.get(ENFORCE_RPM_TPM_ON_MODEL_ADD_SETTING, False)), ) + _raise_if_unroutable_model(model_params.litellm_params) + clean_model_info: Final = ModelInfo( **without_server_derived_pricing(model_params.model_info.model_dump(exclude_none=True)) ) @@ -2504,6 +2585,7 @@ async def update_model( incoming_params=model_params.litellm_params, existing_params=deployment.litellm_params, ) + _raise_if_unroutable_model(model_params.litellm_params) effective_params: Final = _effective_complexity_router_params( model_params.litellm_params, deployment.litellm_params ) diff --git a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py index d1fe88df26c..e79dc4f648b 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py @@ -347,6 +347,264 @@ class TestModelManagementAuthChecks: assert exc_info.value.code == "403" mock_prisma.db.litellm_proxymodeltable.create.assert_not_called() + @pytest.mark.asyncio + async def test_add_new_model_rejects_typesafe_pricing_only_model(self): + from litellm.proxy._types import ProxyException + from litellm.proxy.management_endpoints.model_management_endpoints import ( + add_new_model, + ) + + mock_prisma = MagicMock() + with ( + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), # test-quality-ok: endpoint reads proxy server globals with no injection seam + patch("litellm.proxy.proxy_server.store_model_in_db", True), # test-quality-ok: endpoint reads proxy server globals with no injection seam + patch("litellm.proxy.proxy_server.premium_user", True), # test-quality-ok: endpoint reads proxy server globals with no injection seam + patch( # test-quality-ok: prior auth check needs a live DB; only the unroutable check is under test + "litellm.proxy.management_endpoints.model_management_endpoints.ModelManagementAuthChecks.can_user_make_model_call", + new=AsyncMock(return_value=None), + ), + ): + with pytest.raises(ProxyException) as exc_info: + await add_new_model( + model_params=Deployment( + model_name="jev-via-router", + litellm_params=LiteLLM_Params( + model="typesafe/jev-latest", api_key="os.environ/TYPESAFE_API_KEY" + ), + model_info={"id": "typesafe-test-model"}, + ), + user_api_key_dict=self.admin_user, + ) + assert exc_info.value.code == "400" + assert "typesafe" in exc_info.value.message.lower() + assert "pricing-only" in exc_info.value.message.lower() + mock_prisma.db.litellm_proxymodeltable.create.assert_not_called() + + @pytest.mark.asyncio + async def test_add_new_model_rejects_unroutable_provider(self): + from litellm.proxy._types import ProxyException + from litellm.proxy.management_endpoints.model_management_endpoints import ( + add_new_model, + ) + + mock_prisma = MagicMock() + with ( + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), # test-quality-ok: endpoint reads proxy server globals with no injection seam + patch("litellm.proxy.proxy_server.store_model_in_db", True), # test-quality-ok: endpoint reads proxy server globals with no injection seam + patch("litellm.proxy.proxy_server.premium_user", True), # test-quality-ok: endpoint reads proxy server globals with no injection seam + patch( # test-quality-ok: prior auth check needs a live DB; only the unroutable check is under test + "litellm.proxy.management_endpoints.model_management_endpoints.ModelManagementAuthChecks.can_user_make_model_call", + new=AsyncMock(return_value=None), + ), + ): + with pytest.raises(ProxyException) as exc_info: + await add_new_model( + model_params=Deployment( + model_name="unroutable-model", + litellm_params=LiteLLM_Params( + model="completely_unknown_provider/model-foo" + ), + model_info={"id": "unroutable-test-model"}, + ), + user_api_key_dict=self.admin_user, + ) + assert exc_info.value.code == "400" + assert "unroutable" in exc_info.value.message.lower() or "invalid" in exc_info.value.message.lower() + mock_prisma.db.litellm_proxymodeltable.create.assert_not_called() + + @pytest.mark.asyncio + async def test_patch_model_rejects_typesafe_pricing_only_model(self): + from litellm.proxy._types import ProxyException + from litellm.proxy.management_endpoints.model_management_endpoints import ( + patch_model, + ) + from litellm.types.router import updateLiteLLMParams + + model_id = "typesafe-patch-test" + db_model = Deployment( + model_name="existing-model", + litellm_params=LiteLLM_Params(model="openai/gpt-4o"), + model_info={"id": model_id}, + ) + with ( + patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), # test-quality-ok: endpoint reads proxy server globals with no injection seam + patch("litellm.proxy.proxy_server.llm_router", MagicMock()), # test-quality-ok: endpoint reads proxy server globals with no injection seam + patch("litellm.proxy.proxy_server.store_model_in_db", True), # test-quality-ok: endpoint reads proxy server globals with no injection seam + patch("litellm.proxy.proxy_server.premium_user", True), # test-quality-ok: endpoint reads proxy server globals with no injection seam + patch( # test-quality-ok: stubs the DB row fetch + "litellm.proxy.management_endpoints.model_management_endpoints.get_db_model", + new=AsyncMock(return_value=db_model), + ), + patch( # test-quality-ok: prior auth check needs a live DB + "litellm.proxy.management_endpoints.model_management_endpoints.ModelManagementAuthChecks.can_user_make_model_call", + new=AsyncMock(return_value=None), + ), + patch( # test-quality-ok: asserts the DB write is never reached on rejection + "litellm.proxy.management_endpoints.model_management_endpoints._update_team_model_in_db", + new=AsyncMock(), + ) as mock_update, + ): + with pytest.raises(ProxyException) as exc_info: + await patch_model( + model_id=model_id, + patch_data=updateDeployment( + litellm_params=updateLiteLLMParams(model="typesafe/jev-preview") + ), + user_api_key_dict=self.admin_user, + ) + assert exc_info.value.code == "400" + assert "typesafe" in exc_info.value.message.lower() + mock_update.assert_not_awaited() + + @pytest.mark.asyncio + async def test_update_model_rejects_typesafe_pricing_only_model(self): + from litellm.proxy._types import ProxyException + from litellm.proxy.management_endpoints.model_management_endpoints import ( + update_model, + ) + from litellm.types.router import updateLiteLLMParams + + model_id = "typesafe-update-test" + db_model = Deployment( + model_name="existing-model", + litellm_params=LiteLLM_Params(model="openai/gpt-4o"), + model_info={"id": model_id}, + ) + mock_repo = MagicMock() + mock_repo.table.find_unique = AsyncMock(return_value=db_model) + mock_repo.table.update = AsyncMock() + + with ( + patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), # test-quality-ok: endpoint reads proxy server globals with no injection seam + patch("litellm.proxy.proxy_server.llm_router", MagicMock()), # test-quality-ok: endpoint reads proxy server globals with no injection seam + patch("litellm.proxy.proxy_server.store_model_in_db", True), # test-quality-ok: endpoint reads proxy server globals with no injection seam + patch("litellm.proxy.proxy_server.premium_user", True), # test-quality-ok: endpoint reads proxy server globals with no injection seam + patch( # test-quality-ok: stubs the ModelRepository find_unique call + "litellm.proxy.management_endpoints.model_management_endpoints.ModelRepository", + return_value=mock_repo, + ), + patch( # test-quality-ok: prior auth check needs a live DB + "litellm.proxy.management_endpoints.model_management_endpoints.ModelManagementAuthChecks.can_user_make_model_call", + new=AsyncMock(return_value=None), + ), + ): + with pytest.raises(ProxyException) as exc_info: + await update_model( + model_params=updateDeployment( + litellm_params=updateLiteLLMParams(model="typesafe/jev-1.13.0"), + model_info=ModelInfo(id=model_id), + ), + user_api_key_dict=self.admin_user, + ) + assert exc_info.value.code == "400" + assert "typesafe" in exc_info.value.message.lower() + mock_repo.table.update.assert_not_awaited() + + def test_hypothesis_unroutable_and_typesafe_model_invariants(self): + """Property-based test: any typesafe/* model or unroutable provider must be rejected with 400.""" + from hypothesis import given, strategies as st + from litellm.proxy._types import ProxyException + from litellm.proxy.management_endpoints.model_management_endpoints import ( + _raise_if_unroutable_model, + ) + + @given(st.text(alphabet="abcdefghijklmnopqrstuvwxyz0123456789_-", min_size=1, max_size=30)) + def _check_typesafe(suffix): + with pytest.raises(ProxyException) as exc: + _raise_if_unroutable_model(LiteLLM_Params(model=f"typesafe/{suffix}")) + assert exc.value.code == "400" + assert "typesafe" in exc.value.message.lower() + + _check_typesafe() + + def test_hypothesis_adversarial_input_robustness(self): + """Property-based test: adversarial strings must never trigger unhandled exceptions.""" + from hypothesis import given, strategies as st + from litellm.proxy._types import ProxyException + from litellm.proxy.management_endpoints.model_management_endpoints import ( + _raise_if_unroutable_model, + ) + + executed = [] + + @given(st.text(min_size=0, max_size=50)) + def _check_adversarial(raw_str): + try: + res = _raise_if_unroutable_model(LiteLLM_Params(model=raw_str)) + assert res is None + except ProxyException: + pass # expected rejection for invalid models + executed.append(True) + + _check_adversarial() + assert len(executed) > 0 + + def test_unroutable_guard_allows_valid_standard_models(self): + """Positive control: ensure valid standard models across all major providers pass without rejection.""" + from litellm.proxy.management_endpoints.model_management_endpoints import ( + _raise_if_unroutable_model, + ) + + valid_models = [ + "openai/gpt-4o", + "gpt-4o", + "anthropic/claude-3-5-sonnet-20241022", + "bedrock/anthropic.claude-v2", + "gemini/gemini-1.5-pro", + "azure/gpt-4o", + ] + for model in valid_models: + assert _raise_if_unroutable_model(LiteLLM_Params(model=model)) is None + + def test_unroutable_guard_allows_wildcard_and_prompt_management_callbacks(self): + """Positive control: ensure wildcard routes and prompt-management callbacks bypass provider check.""" + from litellm.proxy.management_endpoints.model_management_endpoints import ( + _raise_if_unroutable_model, + ) + + assert _raise_if_unroutable_model(LiteLLM_Params(model="*")) is None + assert _raise_if_unroutable_model(LiteLLM_Params(model="*/*")) is None + assert _raise_if_unroutable_model(LiteLLM_Params(model="langfuse/test-prompt-template")) is None + + def test_unroutable_guard_supports_dict_and_none_shapes(self): + """Duck-typing test: verify dict, None, and Pydantic shapes behave identically without AttributeError.""" + from litellm.proxy._types import ProxyException + from litellm.proxy.management_endpoints.model_management_endpoints import ( + _raise_if_unroutable_model, + ) + + # None input + assert _raise_if_unroutable_model(None) is None + + # Empty dict + assert _raise_if_unroutable_model({}) is None + + # Valid dict model + assert _raise_if_unroutable_model({"model": "openai/gpt-4o"}) is None + + # Rejected dict typesafe model + with pytest.raises(ProxyException) as exc_info: + _raise_if_unroutable_model({"model": "typesafe/jev-latest"}) + assert exc_info.value.code == "400" + assert "typesafe" in exc_info.value.message.lower() + + # Rejected dict custom_llm_provider + with pytest.raises(ProxyException) as exc_info2: + _raise_if_unroutable_model({"model": "some-model", "custom_llm_provider": "typesafe"}) + assert exc_info2.value.code == "400" + assert "typesafe" in exc_info2.value.message.lower() + + def test_unroutable_guard_runtime_type_annotation_introspection(self): + """Contract test: verify typing.get_type_hints evaluates cleanly for runtime annotation safety.""" + import typing + from litellm.proxy.management_endpoints.model_management_endpoints import ( + _raise_if_unroutable_model, + ) + + hints = typing.get_type_hints(_raise_if_unroutable_model) + assert "litellm_params" in hints + assert "return" in hints + @pytest.mark.asyncio async def test_patch_model_rejects_credential_attach_for_non_admin(self): from litellm.proxy._types import ProxyException