diff --git a/litellm/proxy/management_endpoints/fallback_management_endpoints.py b/litellm/proxy/management_endpoints/fallback_management_endpoints.py index b39afeea86b..477cf44073f 100644 --- a/litellm/proxy/management_endpoints/fallback_management_endpoints.py +++ b/litellm/proxy/management_endpoints/fallback_management_endpoints.py @@ -107,11 +107,14 @@ def _checked_content_policy_rules( def apply_content_policy_fallbacks(llm_router: "Router", rules: object, *, log_skipped_rules: bool) -> None: checked_rules: Final = _checked_content_policy_rules(rules) runtime_rules: Final = tuple(result for _, result in checked_rules if not isinstance(result, ValidationError)) - layered_rules: Final = _layered_over_config_rules( - runtime_rules, - _config_file_rules( - litellm.content_policy_fallbacks # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # untyped module list - ), + layered_rules: Final = ( + _layered_over_config_rules( + runtime_rules, + _config_file_rules( + litellm.content_policy_fallbacks # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # untyped module list + ), + ) + or None ) for rule, result in checked_rules: if log_skipped_rules and isinstance(result, ValidationError): @@ -352,9 +355,7 @@ async def get_fallback( fallback_targets: Final = _FALLBACK_TARGETS.validate_python( get_all_fallbacks(model=model, llm_router=llm_router, fallback_type=fallback_type) ) - fallback_models: Final = [ - name for target in fallback_targets if (name := fallback_target_model_name(target)) is not None - ] + fallback_models: Final = [fallback_target_model_name(target) or model for target in fallback_targets] if not fallback_models: raise HTTPException( diff --git a/litellm/types/proxy/management_endpoints/key_requests.py b/litellm/types/proxy/management_endpoints/key_requests.py index 172357d8ac4..80aa57265a6 100644 --- a/litellm/types/proxy/management_endpoints/key_requests.py +++ b/litellm/types/proxy/management_endpoints/key_requests.py @@ -1,14 +1,14 @@ import enum -from collections.abc import Mapping +from collections.abc import Iterator, Mapping from datetime import datetime from types import MappingProxyType -from typing import Any, Literal # noqa: TID251 # allowed_routes and metadata stay Any for existing callers +from typing import Any, Final, Literal # noqa: TID251 # allowed_routes and metadata stay Any for existing callers from pydantic import Field, PositiveInt, field_validator, model_validator from typing_extensions import TypeIs # noqa: TID251 # narrows the raw payload a before-validator receives from litellm.types.proxy.management_endpoints.request_base import GenerateRequestBase -from litellm.types.router import UpdateRouterConfig +from litellm.types.router import ContentPolicyFallbackRules, UpdateRouterConfig from litellm.types.utils import LiteLLMPydanticObjectBase @@ -17,6 +17,12 @@ class AllowedVectorStoreIndexItem(LiteLLMPydanticObjectBase): index_permissions: list[Literal["read", "write"]] +def _targets_that_are_not_model_names(rules: ContentPolicyFallbackRules) -> Iterator[object]: + for rule in rules: + for targets in rule.values(): + yield from (target for target in targets if not isinstance(target, str)) + + class KeyRequestBase(GenerateRequestBase): key: str | None = None tpd_limit: int | None = None @@ -42,6 +48,18 @@ class KeyRequestBase(GenerateRequestBase): router_settings: UpdateRouterConfig | None = None access_group_ids: list[str] | None = None + @field_validator("router_settings") + @classmethod + def content_policy_fallbacks_name_models( + cls, router_settings: UpdateRouterConfig | None + ) -> UpdateRouterConfig | None: + rules: Final = None if router_settings is None else router_settings.content_policy_fallbacks + if rules is not None and any(_targets_that_are_not_model_names(rules)): + raise ValueError( + 'A key\'s content_policy_fallbacks targets must be model names, e.g. [{"gpt-4": ["gpt-4o"]}]' + ) + return router_settings + class LiteLLMKeyType(str, enum.Enum): """ diff --git a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py index 2fe517b3759..2e8639a1962 100644 --- a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py @@ -13,7 +13,7 @@ import pytest import yaml from fastapi import HTTPException from fastapi.testclient import TestClient -from pydantic import JsonValue +from pydantic import JsonValue, ValidationError import litellm from litellm.constants import PROXY_DB_LOOKUP_STALL_WINDOW_SECONDS @@ -7597,7 +7597,7 @@ async def test_generate_key_with_router_settings(monkeypatch): "num_retries": 3, "model_group_retry_policy": {"gpt-4": {"RateLimitErrorRetries": 5}}, "weights": {"gpt-4": {"weighted-id": 1}}, - "content_policy_fallbacks": [{"gpt-4": ["gpt-4o", {"model": "gpt-4o-mini", "temperature": 0}]}], + "content_policy_fallbacks": [{"gpt-4": ["gpt-4o", "gpt-4o-mini"]}], } request_data = GenerateKeyRequest( @@ -7737,6 +7737,17 @@ async def test_update_key_with_router_settings( assert json.loads(cleared["router_settings"]) == {} +@pytest.mark.parametrize("request_type", [GenerateKeyRequest, UpdateKeyRequest]) +def test_a_key_content_policy_rule_can_only_name_fallback_models( + request_type: type[GenerateKeyRequest] | type[UpdateKeyRequest], +): + with pytest.raises(ValidationError, match="must be model names"): + request_type( + key="sk-key", + router_settings={"content_policy_fallbacks": [{"gpt-4": ["gpt-4o", {"model": "gpt-4o-mini"}]}]}, + ) + + @pytest.mark.asyncio async def test_validate_max_budget(): """ diff --git a/tests/unit/proxy/test_fallback_management_endpoints.py b/tests/unit/proxy/test_fallback_management_endpoints.py index d1647286e80..e3d0d072e0a 100644 --- a/tests/unit/proxy/test_fallback_management_endpoints.py +++ b/tests/unit/proxy/test_fallback_management_endpoints.py @@ -449,6 +449,20 @@ class TestFallbackWritesSeeTheLatestStoredRules: assert response.fallback_models == ["gpt-5.4-mini", "team-fallback"] + @pytest.mark.parametrize( + ("fallback_type", "router_setting"), + [("general", "fallbacks"), ("content_policy", "content_policy_fallbacks")], + ) + async def test_get_names_the_rules_own_model_for_a_dict_target_that_names_none( + self, fallback_type: Literal["general", "content_policy"], router_setting: str + ) -> None: + router: Final = _team_router() + router.update_settings(**{router_setting: [{"team-primary": [{"temperature": 0}, "team-fallback"]}]}) + with patch("litellm.proxy.proxy_server.llm_router", router): + response: Final = await get_fallback("team-primary", fallback_type, MagicMock()) + + assert response.fallback_models == ["team-primary", "team-fallback"] + async def test_get_answers_an_error_for_a_string_valued_rule_instead_of_its_characters(self) -> None: router: Final = _team_router() router.update_settings(fallbacks=[{"team-primary": "team-fallback"}]) diff --git a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py index 1f679a94f37..caf70885e65 100644 --- a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py +++ b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py @@ -5632,6 +5632,21 @@ async def test_db_content_policy_fallbacks_reach_the_router(stored_target: str | assert await _answer_after_content_policy_refusal(router) == "backup answered" +@pytest.mark.asyncio +@pytest.mark.parametrize("stored_rules", [[], [{"primary": "backup"}]]) +async def test_a_saved_content_policy_list_with_no_valid_rule_leaves_refusals_to_the_general_fallbacks( + monkeypatch: pytest.MonkeyPatch, stored_rules: list[object] +): + monkeypatch.setattr(litellm, "content_policy_fallbacks", None) + router: Final = _content_policy_router(fallbacks=[{"primary": ["backup"]}]) + + await proxy_server_module.ProxyConfig()._add_router_settings_from_db_config( + llm_router=router, prisma_client=_router_settings_row(stored_rules) + ) + + assert await _answer_after_content_policy_refusal(router) == "backup answered" + + @pytest.mark.asyncio @pytest.mark.parametrize( ("litellm_settings_rules", "stored_rules", "refused_model"),