mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(proxy): keep general fallbacks when no content-policy rule is saved, name a model-less target's model, and take only model names in key rules
This commit is contained in:
parent
835b953587
commit
0b6def613d
5 changed files with 72 additions and 13 deletions
|
|
@ -107,11 +107,14 @@ def _checked_content_policy_rules(
|
|||
def apply_content_policy_fallbacks(llm_router: "Router", rules: object, *, log_skipped_rules: bool) -> None:
|
||||
checked_rules: Final = _checked_content_policy_rules(rules)
|
||||
runtime_rules: Final = tuple(result for _, result in checked_rules if not isinstance(result, ValidationError))
|
||||
layered_rules: Final = _layered_over_config_rules(
|
||||
runtime_rules,
|
||||
_config_file_rules(
|
||||
litellm.content_policy_fallbacks # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # untyped module list
|
||||
),
|
||||
layered_rules: Final = (
|
||||
_layered_over_config_rules(
|
||||
runtime_rules,
|
||||
_config_file_rules(
|
||||
litellm.content_policy_fallbacks # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # untyped module list
|
||||
),
|
||||
)
|
||||
or None
|
||||
)
|
||||
for rule, result in checked_rules:
|
||||
if log_skipped_rules and isinstance(result, ValidationError):
|
||||
|
|
@ -352,9 +355,7 @@ async def get_fallback(
|
|||
fallback_targets: Final = _FALLBACK_TARGETS.validate_python(
|
||||
get_all_fallbacks(model=model, llm_router=llm_router, fallback_type=fallback_type)
|
||||
)
|
||||
fallback_models: Final = [
|
||||
name for target in fallback_targets if (name := fallback_target_model_name(target)) is not None
|
||||
]
|
||||
fallback_models: Final = [fallback_target_model_name(target) or model for target in fallback_targets]
|
||||
|
||||
if not fallback_models:
|
||||
raise HTTPException(
|
||||
|
|
|
|||
|
|
@ -1,14 +1,14 @@
|
|||
import enum
|
||||
from collections.abc import Mapping
|
||||
from collections.abc import Iterator, Mapping
|
||||
from datetime import datetime
|
||||
from types import MappingProxyType
|
||||
from typing import Any, Literal # noqa: TID251 # allowed_routes and metadata stay Any for existing callers
|
||||
from typing import Any, Final, Literal # noqa: TID251 # allowed_routes and metadata stay Any for existing callers
|
||||
|
||||
from pydantic import Field, PositiveInt, field_validator, model_validator
|
||||
from typing_extensions import TypeIs # noqa: TID251 # narrows the raw payload a before-validator receives
|
||||
|
||||
from litellm.types.proxy.management_endpoints.request_base import GenerateRequestBase
|
||||
from litellm.types.router import UpdateRouterConfig
|
||||
from litellm.types.router import ContentPolicyFallbackRules, UpdateRouterConfig
|
||||
from litellm.types.utils import LiteLLMPydanticObjectBase
|
||||
|
||||
|
||||
|
|
@ -17,6 +17,12 @@ class AllowedVectorStoreIndexItem(LiteLLMPydanticObjectBase):
|
|||
index_permissions: list[Literal["read", "write"]]
|
||||
|
||||
|
||||
def _targets_that_are_not_model_names(rules: ContentPolicyFallbackRules) -> Iterator[object]:
|
||||
for rule in rules:
|
||||
for targets in rule.values():
|
||||
yield from (target for target in targets if not isinstance(target, str))
|
||||
|
||||
|
||||
class KeyRequestBase(GenerateRequestBase):
|
||||
key: str | None = None
|
||||
tpd_limit: int | None = None
|
||||
|
|
@ -42,6 +48,18 @@ class KeyRequestBase(GenerateRequestBase):
|
|||
router_settings: UpdateRouterConfig | None = None
|
||||
access_group_ids: list[str] | None = None
|
||||
|
||||
@field_validator("router_settings")
|
||||
@classmethod
|
||||
def content_policy_fallbacks_name_models(
|
||||
cls, router_settings: UpdateRouterConfig | None
|
||||
) -> UpdateRouterConfig | None:
|
||||
rules: Final = None if router_settings is None else router_settings.content_policy_fallbacks
|
||||
if rules is not None and any(_targets_that_are_not_model_names(rules)):
|
||||
raise ValueError(
|
||||
'A key\'s content_policy_fallbacks targets must be model names, e.g. [{"gpt-4": ["gpt-4o"]}]'
|
||||
)
|
||||
return router_settings
|
||||
|
||||
|
||||
class LiteLLMKeyType(str, enum.Enum):
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ import pytest
|
|||
import yaml
|
||||
from fastapi import HTTPException
|
||||
from fastapi.testclient import TestClient
|
||||
from pydantic import JsonValue
|
||||
from pydantic import JsonValue, ValidationError
|
||||
|
||||
import litellm
|
||||
from litellm.constants import PROXY_DB_LOOKUP_STALL_WINDOW_SECONDS
|
||||
|
|
@ -7597,7 +7597,7 @@ async def test_generate_key_with_router_settings(monkeypatch):
|
|||
"num_retries": 3,
|
||||
"model_group_retry_policy": {"gpt-4": {"RateLimitErrorRetries": 5}},
|
||||
"weights": {"gpt-4": {"weighted-id": 1}},
|
||||
"content_policy_fallbacks": [{"gpt-4": ["gpt-4o", {"model": "gpt-4o-mini", "temperature": 0}]}],
|
||||
"content_policy_fallbacks": [{"gpt-4": ["gpt-4o", "gpt-4o-mini"]}],
|
||||
}
|
||||
|
||||
request_data = GenerateKeyRequest(
|
||||
|
|
@ -7737,6 +7737,17 @@ async def test_update_key_with_router_settings(
|
|||
assert json.loads(cleared["router_settings"]) == {}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("request_type", [GenerateKeyRequest, UpdateKeyRequest])
|
||||
def test_a_key_content_policy_rule_can_only_name_fallback_models(
|
||||
request_type: type[GenerateKeyRequest] | type[UpdateKeyRequest],
|
||||
):
|
||||
with pytest.raises(ValidationError, match="must be model names"):
|
||||
request_type(
|
||||
key="sk-key",
|
||||
router_settings={"content_policy_fallbacks": [{"gpt-4": ["gpt-4o", {"model": "gpt-4o-mini"}]}]},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_validate_max_budget():
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -449,6 +449,20 @@ class TestFallbackWritesSeeTheLatestStoredRules:
|
|||
|
||||
assert response.fallback_models == ["gpt-5.4-mini", "team-fallback"]
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("fallback_type", "router_setting"),
|
||||
[("general", "fallbacks"), ("content_policy", "content_policy_fallbacks")],
|
||||
)
|
||||
async def test_get_names_the_rules_own_model_for_a_dict_target_that_names_none(
|
||||
self, fallback_type: Literal["general", "content_policy"], router_setting: str
|
||||
) -> None:
|
||||
router: Final = _team_router()
|
||||
router.update_settings(**{router_setting: [{"team-primary": [{"temperature": 0}, "team-fallback"]}]})
|
||||
with patch("litellm.proxy.proxy_server.llm_router", router):
|
||||
response: Final = await get_fallback("team-primary", fallback_type, MagicMock())
|
||||
|
||||
assert response.fallback_models == ["team-primary", "team-fallback"]
|
||||
|
||||
async def test_get_answers_an_error_for_a_string_valued_rule_instead_of_its_characters(self) -> None:
|
||||
router: Final = _team_router()
|
||||
router.update_settings(fallbacks=[{"team-primary": "team-fallback"}])
|
||||
|
|
|
|||
|
|
@ -5632,6 +5632,21 @@ async def test_db_content_policy_fallbacks_reach_the_router(stored_target: str |
|
|||
assert await _answer_after_content_policy_refusal(router) == "backup answered"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("stored_rules", [[], [{"primary": "backup"}]])
|
||||
async def test_a_saved_content_policy_list_with_no_valid_rule_leaves_refusals_to_the_general_fallbacks(
|
||||
monkeypatch: pytest.MonkeyPatch, stored_rules: list[object]
|
||||
):
|
||||
monkeypatch.setattr(litellm, "content_policy_fallbacks", None)
|
||||
router: Final = _content_policy_router(fallbacks=[{"primary": ["backup"]}])
|
||||
|
||||
await proxy_server_module.ProxyConfig()._add_router_settings_from_db_config(
|
||||
llm_router=router, prisma_client=_router_settings_row(stored_rules)
|
||||
)
|
||||
|
||||
assert await _answer_after_content_policy_refusal(router) == "backup answered"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("litellm_settings_rules", "stored_rules", "refused_model"),
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue