mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(proxy): reject unroutable and pricing-only TypeSafe models before DB write (#41714)
- Validate model routability in model management write endpoints (add_new_model, patch_model, update_model) prior to DB persistence
- Reject typesafe/* models with HTTP 400 explaining that TypeSafe models are pricing-only, directing users to /typesafe/{endpoint} pass-through and classifier_type: jev
- Reject unrecognized providers before persisting rows to LiteLLM_ProxyModelTable, preventing corrupted rows that fail router reload and cause 500 degraded serving errors
- Add comprehensive hermetic unit tests, positive controls, duck-typing, runtime annotation contracts, and Hypothesis property-based fuzzing
Fixes #41714
This commit is contained in:
parent
db37977307
commit
edb2961e7d
2 changed files with 340 additions and 0 deletions
|
|
@ -328,6 +328,84 @@ def _raise_on_strategy_router_write_violation(
|
|||
)
|
||||
|
||||
|
||||
def _raise_if_unroutable_model(litellm_params: GenericLiteLLMParams | None) -> None:
|
||||
"""Validate that the deployment specifies a routable model and provider before persisting to DB.
|
||||
|
||||
Rejects models that are pricing-only (such as 'typesafe/*') or whose provider cannot be resolved
|
||||
by the router, preventing corrupted rows from being committed to LiteLLM_ProxyModelTable that
|
||||
later fail router reload and trigger 500 degraded serving errors.
|
||||
"""
|
||||
if litellm_params is None:
|
||||
return
|
||||
|
||||
raw_model = (
|
||||
getattr(litellm_params, "model", None) if not isinstance(litellm_params, dict) else litellm_params.get("model")
|
||||
)
|
||||
if not raw_model or not isinstance(raw_model, str):
|
||||
return
|
||||
|
||||
custom_llm_provider = (
|
||||
getattr(litellm_params, "custom_llm_provider", None)
|
||||
if not isinstance(litellm_params, dict)
|
||||
else litellm_params.get("custom_llm_provider")
|
||||
)
|
||||
|
||||
# 1. TypeSafe models in LiteLLM are pricing-only for passthrough / auto-router classifier
|
||||
if raw_model.startswith("typesafe/") or custom_llm_provider == "typesafe":
|
||||
raise ProxyException(
|
||||
message=(
|
||||
f"Model '{raw_model}' uses provider 'typesafe'. TypeSafe models in LiteLLM are pricing-only "
|
||||
"and cannot be added as a routable model deployment. Use the '/typesafe/{endpoint}' pass-through "
|
||||
"route (https://docs.litellm.ai/docs/pass_through/typesafe) or configure 'classifier_type: jev' "
|
||||
"in your auto-router settings."
|
||||
),
|
||||
type=ProxyErrorTypes.validation_error.value,
|
||||
code=status.HTTP_400_BAD_REQUEST,
|
||||
param="litellm_params.model",
|
||||
)
|
||||
|
||||
# 2. Skip prompt-management callbacks, wildcards, and special router routes
|
||||
if "/" in raw_model:
|
||||
split_prefix = raw_model.split("/")[0]
|
||||
if split_prefix in getattr(litellm, "_known_custom_logger_compatible_callbacks", ()):
|
||||
return
|
||||
|
||||
if raw_model.startswith("*"):
|
||||
return
|
||||
|
||||
# 3. Validate provider resolution through get_llm_provider
|
||||
try:
|
||||
api_base = (
|
||||
getattr(litellm_params, "api_base", None)
|
||||
if not isinstance(litellm_params, dict)
|
||||
else litellm_params.get("api_base")
|
||||
)
|
||||
_, resolved_provider, _, _ = litellm.get_llm_provider(
|
||||
model=raw_model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
api_base=api_base,
|
||||
)
|
||||
from litellm.router import JSONProviderRegistry, is_registered_custom_provider
|
||||
|
||||
if (
|
||||
resolved_provider not in litellm.provider_list
|
||||
and not JSONProviderRegistry.exists(resolved_provider)
|
||||
and not is_registered_custom_provider(resolved_provider)
|
||||
):
|
||||
raise ValueError(
|
||||
f"Provider '{resolved_provider}' is not supported. Supported providers: {litellm.provider_list}"
|
||||
)
|
||||
except Exception as e:
|
||||
if isinstance(e, ProxyException):
|
||||
raise e
|
||||
raise ProxyException(
|
||||
message=f"Invalid or unroutable model '{raw_model}': {e}",
|
||||
type=ProxyErrorTypes.validation_error.value,
|
||||
code=status.HTTP_400_BAD_REQUEST,
|
||||
param="litellm_params.model",
|
||||
)
|
||||
|
||||
|
||||
AUTO_ROUTER_CAPABILITY_SLOT_LOCK_KEY: Final = 5_872_301
|
||||
_CAPABILITY_LOCK_SQL: Final = "SELECT 1 AS locked FROM pg_advisory_xact_lock($1)"
|
||||
_STORED_LITELLM_PARAMS_SQL: Final = (
|
||||
|
|
@ -1087,6 +1165,7 @@ async def patch_model(
|
|||
incoming_params=patch_data.litellm_params,
|
||||
existing_params=db_model.litellm_params,
|
||||
)
|
||||
_raise_if_unroutable_model(patch_data.litellm_params)
|
||||
|
||||
effective_params: Final = _effective_complexity_router_params(
|
||||
patch_data.litellm_params, db_model.litellm_params
|
||||
|
|
@ -2306,6 +2385,8 @@ async def add_new_model(
|
|||
enforced=bool(general_settings.get(ENFORCE_RPM_TPM_ON_MODEL_ADD_SETTING, False)),
|
||||
)
|
||||
|
||||
_raise_if_unroutable_model(model_params.litellm_params)
|
||||
|
||||
clean_model_info: Final = ModelInfo(
|
||||
**without_server_derived_pricing(model_params.model_info.model_dump(exclude_none=True))
|
||||
)
|
||||
|
|
@ -2504,6 +2585,7 @@ async def update_model(
|
|||
incoming_params=model_params.litellm_params,
|
||||
existing_params=deployment.litellm_params,
|
||||
)
|
||||
_raise_if_unroutable_model(model_params.litellm_params)
|
||||
effective_params: Final = _effective_complexity_router_params(
|
||||
model_params.litellm_params, deployment.litellm_params
|
||||
)
|
||||
|
|
|
|||
|
|
@ -347,6 +347,264 @@ class TestModelManagementAuthChecks:
|
|||
assert exc_info.value.code == "403"
|
||||
mock_prisma.db.litellm_proxymodeltable.create.assert_not_called()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_new_model_rejects_typesafe_pricing_only_model(self):
|
||||
from litellm.proxy._types import ProxyException
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import (
|
||||
add_new_model,
|
||||
)
|
||||
|
||||
mock_prisma = MagicMock()
|
||||
with (
|
||||
patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), # test-quality-ok: endpoint reads proxy server globals with no injection seam
|
||||
patch("litellm.proxy.proxy_server.store_model_in_db", True), # test-quality-ok: endpoint reads proxy server globals with no injection seam
|
||||
patch("litellm.proxy.proxy_server.premium_user", True), # test-quality-ok: endpoint reads proxy server globals with no injection seam
|
||||
patch( # test-quality-ok: prior auth check needs a live DB; only the unroutable check is under test
|
||||
"litellm.proxy.management_endpoints.model_management_endpoints.ModelManagementAuthChecks.can_user_make_model_call",
|
||||
new=AsyncMock(return_value=None),
|
||||
),
|
||||
):
|
||||
with pytest.raises(ProxyException) as exc_info:
|
||||
await add_new_model(
|
||||
model_params=Deployment(
|
||||
model_name="jev-via-router",
|
||||
litellm_params=LiteLLM_Params(
|
||||
model="typesafe/jev-latest", api_key="os.environ/TYPESAFE_API_KEY"
|
||||
),
|
||||
model_info={"id": "typesafe-test-model"},
|
||||
),
|
||||
user_api_key_dict=self.admin_user,
|
||||
)
|
||||
assert exc_info.value.code == "400"
|
||||
assert "typesafe" in exc_info.value.message.lower()
|
||||
assert "pricing-only" in exc_info.value.message.lower()
|
||||
mock_prisma.db.litellm_proxymodeltable.create.assert_not_called()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_new_model_rejects_unroutable_provider(self):
|
||||
from litellm.proxy._types import ProxyException
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import (
|
||||
add_new_model,
|
||||
)
|
||||
|
||||
mock_prisma = MagicMock()
|
||||
with (
|
||||
patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), # test-quality-ok: endpoint reads proxy server globals with no injection seam
|
||||
patch("litellm.proxy.proxy_server.store_model_in_db", True), # test-quality-ok: endpoint reads proxy server globals with no injection seam
|
||||
patch("litellm.proxy.proxy_server.premium_user", True), # test-quality-ok: endpoint reads proxy server globals with no injection seam
|
||||
patch( # test-quality-ok: prior auth check needs a live DB; only the unroutable check is under test
|
||||
"litellm.proxy.management_endpoints.model_management_endpoints.ModelManagementAuthChecks.can_user_make_model_call",
|
||||
new=AsyncMock(return_value=None),
|
||||
),
|
||||
):
|
||||
with pytest.raises(ProxyException) as exc_info:
|
||||
await add_new_model(
|
||||
model_params=Deployment(
|
||||
model_name="unroutable-model",
|
||||
litellm_params=LiteLLM_Params(
|
||||
model="completely_unknown_provider/model-foo"
|
||||
),
|
||||
model_info={"id": "unroutable-test-model"},
|
||||
),
|
||||
user_api_key_dict=self.admin_user,
|
||||
)
|
||||
assert exc_info.value.code == "400"
|
||||
assert "unroutable" in exc_info.value.message.lower() or "invalid" in exc_info.value.message.lower()
|
||||
mock_prisma.db.litellm_proxymodeltable.create.assert_not_called()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_patch_model_rejects_typesafe_pricing_only_model(self):
|
||||
from litellm.proxy._types import ProxyException
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import (
|
||||
patch_model,
|
||||
)
|
||||
from litellm.types.router import updateLiteLLMParams
|
||||
|
||||
model_id = "typesafe-patch-test"
|
||||
db_model = Deployment(
|
||||
model_name="existing-model",
|
||||
litellm_params=LiteLLM_Params(model="openai/gpt-4o"),
|
||||
model_info={"id": model_id},
|
||||
)
|
||||
with (
|
||||
patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), # test-quality-ok: endpoint reads proxy server globals with no injection seam
|
||||
patch("litellm.proxy.proxy_server.llm_router", MagicMock()), # test-quality-ok: endpoint reads proxy server globals with no injection seam
|
||||
patch("litellm.proxy.proxy_server.store_model_in_db", True), # test-quality-ok: endpoint reads proxy server globals with no injection seam
|
||||
patch("litellm.proxy.proxy_server.premium_user", True), # test-quality-ok: endpoint reads proxy server globals with no injection seam
|
||||
patch( # test-quality-ok: stubs the DB row fetch
|
||||
"litellm.proxy.management_endpoints.model_management_endpoints.get_db_model",
|
||||
new=AsyncMock(return_value=db_model),
|
||||
),
|
||||
patch( # test-quality-ok: prior auth check needs a live DB
|
||||
"litellm.proxy.management_endpoints.model_management_endpoints.ModelManagementAuthChecks.can_user_make_model_call",
|
||||
new=AsyncMock(return_value=None),
|
||||
),
|
||||
patch( # test-quality-ok: asserts the DB write is never reached on rejection
|
||||
"litellm.proxy.management_endpoints.model_management_endpoints._update_team_model_in_db",
|
||||
new=AsyncMock(),
|
||||
) as mock_update,
|
||||
):
|
||||
with pytest.raises(ProxyException) as exc_info:
|
||||
await patch_model(
|
||||
model_id=model_id,
|
||||
patch_data=updateDeployment(
|
||||
litellm_params=updateLiteLLMParams(model="typesafe/jev-preview")
|
||||
),
|
||||
user_api_key_dict=self.admin_user,
|
||||
)
|
||||
assert exc_info.value.code == "400"
|
||||
assert "typesafe" in exc_info.value.message.lower()
|
||||
mock_update.assert_not_awaited()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_model_rejects_typesafe_pricing_only_model(self):
|
||||
from litellm.proxy._types import ProxyException
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import (
|
||||
update_model,
|
||||
)
|
||||
from litellm.types.router import updateLiteLLMParams
|
||||
|
||||
model_id = "typesafe-update-test"
|
||||
db_model = Deployment(
|
||||
model_name="existing-model",
|
||||
litellm_params=LiteLLM_Params(model="openai/gpt-4o"),
|
||||
model_info={"id": model_id},
|
||||
)
|
||||
mock_repo = MagicMock()
|
||||
mock_repo.table.find_unique = AsyncMock(return_value=db_model)
|
||||
mock_repo.table.update = AsyncMock()
|
||||
|
||||
with (
|
||||
patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), # test-quality-ok: endpoint reads proxy server globals with no injection seam
|
||||
patch("litellm.proxy.proxy_server.llm_router", MagicMock()), # test-quality-ok: endpoint reads proxy server globals with no injection seam
|
||||
patch("litellm.proxy.proxy_server.store_model_in_db", True), # test-quality-ok: endpoint reads proxy server globals with no injection seam
|
||||
patch("litellm.proxy.proxy_server.premium_user", True), # test-quality-ok: endpoint reads proxy server globals with no injection seam
|
||||
patch( # test-quality-ok: stubs the ModelRepository find_unique call
|
||||
"litellm.proxy.management_endpoints.model_management_endpoints.ModelRepository",
|
||||
return_value=mock_repo,
|
||||
),
|
||||
patch( # test-quality-ok: prior auth check needs a live DB
|
||||
"litellm.proxy.management_endpoints.model_management_endpoints.ModelManagementAuthChecks.can_user_make_model_call",
|
||||
new=AsyncMock(return_value=None),
|
||||
),
|
||||
):
|
||||
with pytest.raises(ProxyException) as exc_info:
|
||||
await update_model(
|
||||
model_params=updateDeployment(
|
||||
litellm_params=updateLiteLLMParams(model="typesafe/jev-1.13.0"),
|
||||
model_info=ModelInfo(id=model_id),
|
||||
),
|
||||
user_api_key_dict=self.admin_user,
|
||||
)
|
||||
assert exc_info.value.code == "400"
|
||||
assert "typesafe" in exc_info.value.message.lower()
|
||||
mock_repo.table.update.assert_not_awaited()
|
||||
|
||||
def test_hypothesis_unroutable_and_typesafe_model_invariants(self):
|
||||
"""Property-based test: any typesafe/* model or unroutable provider must be rejected with 400."""
|
||||
from hypothesis import given, strategies as st
|
||||
from litellm.proxy._types import ProxyException
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import (
|
||||
_raise_if_unroutable_model,
|
||||
)
|
||||
|
||||
@given(st.text(alphabet="abcdefghijklmnopqrstuvwxyz0123456789_-", min_size=1, max_size=30))
|
||||
def _check_typesafe(suffix):
|
||||
with pytest.raises(ProxyException) as exc:
|
||||
_raise_if_unroutable_model(LiteLLM_Params(model=f"typesafe/{suffix}"))
|
||||
assert exc.value.code == "400"
|
||||
assert "typesafe" in exc.value.message.lower()
|
||||
|
||||
_check_typesafe()
|
||||
|
||||
def test_hypothesis_adversarial_input_robustness(self):
|
||||
"""Property-based test: adversarial strings must never trigger unhandled exceptions."""
|
||||
from hypothesis import given, strategies as st
|
||||
from litellm.proxy._types import ProxyException
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import (
|
||||
_raise_if_unroutable_model,
|
||||
)
|
||||
|
||||
executed = []
|
||||
|
||||
@given(st.text(min_size=0, max_size=50))
|
||||
def _check_adversarial(raw_str):
|
||||
try:
|
||||
res = _raise_if_unroutable_model(LiteLLM_Params(model=raw_str))
|
||||
assert res is None
|
||||
except ProxyException:
|
||||
pass # expected rejection for invalid models
|
||||
executed.append(True)
|
||||
|
||||
_check_adversarial()
|
||||
assert len(executed) > 0
|
||||
|
||||
def test_unroutable_guard_allows_valid_standard_models(self):
|
||||
"""Positive control: ensure valid standard models across all major providers pass without rejection."""
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import (
|
||||
_raise_if_unroutable_model,
|
||||
)
|
||||
|
||||
valid_models = [
|
||||
"openai/gpt-4o",
|
||||
"gpt-4o",
|
||||
"anthropic/claude-3-5-sonnet-20241022",
|
||||
"bedrock/anthropic.claude-v2",
|
||||
"gemini/gemini-1.5-pro",
|
||||
"azure/gpt-4o",
|
||||
]
|
||||
for model in valid_models:
|
||||
assert _raise_if_unroutable_model(LiteLLM_Params(model=model)) is None
|
||||
|
||||
def test_unroutable_guard_allows_wildcard_and_prompt_management_callbacks(self):
|
||||
"""Positive control: ensure wildcard routes and prompt-management callbacks bypass provider check."""
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import (
|
||||
_raise_if_unroutable_model,
|
||||
)
|
||||
|
||||
assert _raise_if_unroutable_model(LiteLLM_Params(model="*")) is None
|
||||
assert _raise_if_unroutable_model(LiteLLM_Params(model="*/*")) is None
|
||||
assert _raise_if_unroutable_model(LiteLLM_Params(model="langfuse/test-prompt-template")) is None
|
||||
|
||||
def test_unroutable_guard_supports_dict_and_none_shapes(self):
|
||||
"""Duck-typing test: verify dict, None, and Pydantic shapes behave identically without AttributeError."""
|
||||
from litellm.proxy._types import ProxyException
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import (
|
||||
_raise_if_unroutable_model,
|
||||
)
|
||||
|
||||
# None input
|
||||
assert _raise_if_unroutable_model(None) is None
|
||||
|
||||
# Empty dict
|
||||
assert _raise_if_unroutable_model({}) is None
|
||||
|
||||
# Valid dict model
|
||||
assert _raise_if_unroutable_model({"model": "openai/gpt-4o"}) is None
|
||||
|
||||
# Rejected dict typesafe model
|
||||
with pytest.raises(ProxyException) as exc_info:
|
||||
_raise_if_unroutable_model({"model": "typesafe/jev-latest"})
|
||||
assert exc_info.value.code == "400"
|
||||
assert "typesafe" in exc_info.value.message.lower()
|
||||
|
||||
# Rejected dict custom_llm_provider
|
||||
with pytest.raises(ProxyException) as exc_info2:
|
||||
_raise_if_unroutable_model({"model": "some-model", "custom_llm_provider": "typesafe"})
|
||||
assert exc_info2.value.code == "400"
|
||||
assert "typesafe" in exc_info2.value.message.lower()
|
||||
|
||||
def test_unroutable_guard_runtime_type_annotation_introspection(self):
|
||||
"""Contract test: verify typing.get_type_hints evaluates cleanly for runtime annotation safety."""
|
||||
import typing
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import (
|
||||
_raise_if_unroutable_model,
|
||||
)
|
||||
|
||||
hints = typing.get_type_hints(_raise_if_unroutable_model)
|
||||
assert "litellm_params" in hints
|
||||
assert "return" in hints
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_patch_model_rejects_credential_attach_for_non_admin(self):
|
||||
from litellm.proxy._types import ProxyException
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue