fix(proxy): keep general fallbacks when no content-policy rule is saved, name a model-less target's model, and take only model names in key rules

This commit is contained in:
mateo-berri 2026-10-29 23:28:39 -07:00
parent 835b953587
commit 0b6def613d
5 changed files with 72 additions and 13 deletions

View file

@ -107,11 +107,14 @@ def _checked_content_policy_rules(
def apply_content_policy_fallbacks(llm_router: "Router", rules: object, *, log_skipped_rules: bool) -> None:
checked_rules: Final = _checked_content_policy_rules(rules)
runtime_rules: Final = tuple(result for _, result in checked_rules if not isinstance(result, ValidationError))
layered_rules: Final = _layered_over_config_rules(
runtime_rules,
_config_file_rules(
litellm.content_policy_fallbacks # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # untyped module list
),
layered_rules: Final = (
_layered_over_config_rules(
runtime_rules,
_config_file_rules(
litellm.content_policy_fallbacks # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # untyped module list
),
)
or None
)
for rule, result in checked_rules:
if log_skipped_rules and isinstance(result, ValidationError):
@ -352,9 +355,7 @@ async def get_fallback(
fallback_targets: Final = _FALLBACK_TARGETS.validate_python(
get_all_fallbacks(model=model, llm_router=llm_router, fallback_type=fallback_type)
)
fallback_models: Final = [
name for target in fallback_targets if (name := fallback_target_model_name(target)) is not None
]
fallback_models: Final = [fallback_target_model_name(target) or model for target in fallback_targets]
if not fallback_models:
raise HTTPException(

View file

@ -1,14 +1,14 @@
import enum
from collections.abc import Mapping
from collections.abc import Iterator, Mapping
from datetime import datetime
from types import MappingProxyType
from typing import Any, Literal # noqa: TID251 # allowed_routes and metadata stay Any for existing callers
from typing import Any, Final, Literal # noqa: TID251 # allowed_routes and metadata stay Any for existing callers
from pydantic import Field, PositiveInt, field_validator, model_validator
from typing_extensions import TypeIs # noqa: TID251 # narrows the raw payload a before-validator receives
from litellm.types.proxy.management_endpoints.request_base import GenerateRequestBase
from litellm.types.router import UpdateRouterConfig
from litellm.types.router import ContentPolicyFallbackRules, UpdateRouterConfig
from litellm.types.utils import LiteLLMPydanticObjectBase
@ -17,6 +17,12 @@ class AllowedVectorStoreIndexItem(LiteLLMPydanticObjectBase):
index_permissions: list[Literal["read", "write"]]
def _targets_that_are_not_model_names(rules: ContentPolicyFallbackRules) -> Iterator[object]:
for rule in rules:
for targets in rule.values():
yield from (target for target in targets if not isinstance(target, str))
class KeyRequestBase(GenerateRequestBase):
key: str | None = None
tpd_limit: int | None = None
@ -42,6 +48,18 @@ class KeyRequestBase(GenerateRequestBase):
router_settings: UpdateRouterConfig | None = None
access_group_ids: list[str] | None = None
@field_validator("router_settings")
@classmethod
def content_policy_fallbacks_name_models(
cls, router_settings: UpdateRouterConfig | None
) -> UpdateRouterConfig | None:
rules: Final = None if router_settings is None else router_settings.content_policy_fallbacks
if rules is not None and any(_targets_that_are_not_model_names(rules)):
raise ValueError(
'A key\'s content_policy_fallbacks targets must be model names, e.g. [{"gpt-4": ["gpt-4o"]}]'
)
return router_settings
class LiteLLMKeyType(str, enum.Enum):
"""

View file

@ -13,7 +13,7 @@ import pytest
import yaml
from fastapi import HTTPException
from fastapi.testclient import TestClient
from pydantic import JsonValue
from pydantic import JsonValue, ValidationError
import litellm
from litellm.constants import PROXY_DB_LOOKUP_STALL_WINDOW_SECONDS
@ -7597,7 +7597,7 @@ async def test_generate_key_with_router_settings(monkeypatch):
"num_retries": 3,
"model_group_retry_policy": {"gpt-4": {"RateLimitErrorRetries": 5}},
"weights": {"gpt-4": {"weighted-id": 1}},
"content_policy_fallbacks": [{"gpt-4": ["gpt-4o", {"model": "gpt-4o-mini", "temperature": 0}]}],
"content_policy_fallbacks": [{"gpt-4": ["gpt-4o", "gpt-4o-mini"]}],
}
request_data = GenerateKeyRequest(
@ -7737,6 +7737,17 @@ async def test_update_key_with_router_settings(
assert json.loads(cleared["router_settings"]) == {}
@pytest.mark.parametrize("request_type", [GenerateKeyRequest, UpdateKeyRequest])
def test_a_key_content_policy_rule_can_only_name_fallback_models(
request_type: type[GenerateKeyRequest] | type[UpdateKeyRequest],
):
with pytest.raises(ValidationError, match="must be model names"):
request_type(
key="sk-key",
router_settings={"content_policy_fallbacks": [{"gpt-4": ["gpt-4o", {"model": "gpt-4o-mini"}]}]},
)
@pytest.mark.asyncio
async def test_validate_max_budget():
"""

View file

@ -449,6 +449,20 @@ class TestFallbackWritesSeeTheLatestStoredRules:
assert response.fallback_models == ["gpt-5.4-mini", "team-fallback"]
@pytest.mark.parametrize(
("fallback_type", "router_setting"),
[("general", "fallbacks"), ("content_policy", "content_policy_fallbacks")],
)
async def test_get_names_the_rules_own_model_for_a_dict_target_that_names_none(
self, fallback_type: Literal["general", "content_policy"], router_setting: str
) -> None:
router: Final = _team_router()
router.update_settings(**{router_setting: [{"team-primary": [{"temperature": 0}, "team-fallback"]}]})
with patch("litellm.proxy.proxy_server.llm_router", router):
response: Final = await get_fallback("team-primary", fallback_type, MagicMock())
assert response.fallback_models == ["team-primary", "team-fallback"]
async def test_get_answers_an_error_for_a_string_valued_rule_instead_of_its_characters(self) -> None:
router: Final = _team_router()
router.update_settings(fallbacks=[{"team-primary": "team-fallback"}])

View file

@ -5632,6 +5632,21 @@ async def test_db_content_policy_fallbacks_reach_the_router(stored_target: str |
assert await _answer_after_content_policy_refusal(router) == "backup answered"
@pytest.mark.asyncio
@pytest.mark.parametrize("stored_rules", [[], [{"primary": "backup"}]])
async def test_a_saved_content_policy_list_with_no_valid_rule_leaves_refusals_to_the_general_fallbacks(
monkeypatch: pytest.MonkeyPatch, stored_rules: list[object]
):
monkeypatch.setattr(litellm, "content_policy_fallbacks", None)
router: Final = _content_policy_router(fallbacks=[{"primary": ["backup"]}])
await proxy_server_module.ProxyConfig()._add_router_settings_from_db_config(
llm_router=router, prisma_client=_router_settings_row(stored_rules)
)
assert await _answer_after_content_policy_refusal(router) == "backup answered"
@pytest.mark.asyncio
@pytest.mark.parametrize(
("litellm_settings_rules", "stored_rules", "refused_model"),