This commit is contained in:
devin-ai-integration[bot] 2026-10-30 02:45:47 +00:00 • committed by GitHub
commit 6dabc68b4c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 488 additions and 28 deletions

View file

@ -67,6 +67,7 @@ RUNTIME_UPDATABLE_ROUTER_SETTINGS: Final[frozenset[str]] = frozenset(
"retry_after",
"fallbacks",
"context_window_fallbacks",
"content_policy_fallbacks",
"retry_policy",
"model_group_retry_policy",
"model_group_alias",

View file

@ -3138,9 +3138,9 @@ def _fallback_lists(
) -> tuple[Sequence[object], ...]:
"""Every fallback list the router can route this request through: the three
router-level lists, the same three keys from the request body (which replace
the router's when present, so the union is the safe superset), and the
key/team ``router_settings.fallbacks`` the proxy prefers over router-level
fallbacks in ``_configured_fallbacks``."""
the router's when present, so the union is the safe superset), and the same
three keys from key/team ``router_settings``, which the proxy forwards into
the request after this discovery runs."""
lists: Final[list[Sequence[object]]] = [] # mutable-ok: accumulates each fallback list found
configured: Final = llm_router.settings()
for router_list in (
@ -3155,11 +3155,12 @@ def _fallback_lists(
if isinstance(entries, list):
lists.append(cast("Sequence[object]", entries)) # cast-ok: request fallback lists hold mixed entry shapes
for settings in (router_settings, team_router_settings):
key_fallbacks: object = settings.get("fallbacks") if settings is not None else None
if isinstance(key_fallbacks, list):
lists.append(
cast("Sequence[object]", key_fallbacks) # cast-ok: settings fallback lists hold mixed entry shapes
)
for key in _REQUEST_FALLBACK_KEYS:
key_fallbacks: object = settings.get(key) if settings is not None else None
if isinstance(key_fallbacks, list):
lists.append(
cast("Sequence[object]", key_fallbacks) # cast-ok: settings fallback lists hold mixed entry shapes
)
return tuple(lists)

View file

@ -14,10 +14,12 @@ import json
from collections.abc import Mapping, Sequence
from typing import TYPE_CHECKING, Annotated, Final, Literal
from pydantic import Field, JsonValue, TypeAdapter
from pydantic import Field, JsonValue, TypeAdapter, ValidationError
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.auth.auth_utils import fallback_target_model_name
from litellm.proxy.auth.model_checks import get_all_fallbacks
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.utils import PrismaClient, invalidate_config_param
@ -26,6 +28,7 @@ if TYPE_CHECKING:
from fastapi import APIRouter, Depends, HTTPException, status
from litellm.proxy.proxy_server import ProxyConfig
from litellm.router import Router
else:
try:
from fastapi import APIRouter, Depends, HTTPException, status
@ -41,6 +44,7 @@ from litellm.types.management_endpoints.router_settings_endpoints import (
FallbackGetResponse,
FallbackResponse,
)
from litellm.types.router import ContentPolicyFallbackRule, ContentPolicyFallbackRules, FallbackTargets
router: Final = APIRouter()
@ -50,12 +54,80 @@ StoredFallback = Annotated[FallbackRule | dict[str, object] | str, Field(union_m
_STORED_FALLBACKS: Final = TypeAdapter(list[StoredFallback])
_JSON_RULES: Final = TypeAdapter(list[JsonValue])
_STORED_ROUTER_SETTINGS: Final = TypeAdapter(dict[str, JsonValue])
_OPTIONAL_JSON_RULES: Final[TypeAdapter[Sequence[JsonValue] | None]] = TypeAdapter(Sequence[JsonValue] | None)
_CONTENT_POLICY_FALLBACK_RULE: Final[TypeAdapter[ContentPolicyFallbackRule]] = TypeAdapter(ContentPolicyFallbackRule)
_FALLBACK_TARGETS: Final[TypeAdapter[FallbackTargets]] = TypeAdapter(FallbackTargets)
def _rule_covers(entry: StoredFallback, model: str) -> bool:
return isinstance(entry, dict) and model in entry
def _layered_over_config_rules(
runtime_rules: ContentPolicyFallbackRules, config_rules: Sequence[Mapping[str, JsonValue]]
) -> Sequence[Mapping[str, object]]:
covered: Final = frozenset[str]().union(*runtime_rules)
return [*runtime_rules, *(rule for rule in config_rules if covered.isdisjoint(rule))]
def _config_file_rules(rules: object) -> Sequence[Mapping[str, JsonValue]]:
try:
candidates: Final = _OPTIONAL_JSON_RULES.validate_python(rules) or ()
except ValidationError:
return ()
return tuple(rule for rule in candidates if isinstance(rule, Mapping))
def _log_ignored_content_policy_rules(rules: object, error: ValidationError) -> None:
verbose_proxy_logger.error(
"Ignoring invalid content_policy_fallbacks %r from the DB, every other rule still applies. "
"Fix it through POST /fallback or /config/update: %s",
rules,
error,
)
def _checked_content_policy_rule(rule: JsonValue) -> ContentPolicyFallbackRule | ValidationError:
try:
return _CONTENT_POLICY_FALLBACK_RULE.validate_python(rule)
except ValidationError as invalid_rule:
return invalid_rule
def _checked_content_policy_rules(
rules: object,
) -> Sequence[tuple[object, ContentPolicyFallbackRule | ValidationError]]:
try:
candidates: Final = _OPTIONAL_JSON_RULES.validate_python(rules) or ()
except ValidationError as not_a_rule_list:
return ((rules, not_a_rule_list),)
return tuple((candidate, _checked_content_policy_rule(candidate)) for candidate in candidates)
def apply_content_policy_fallbacks(llm_router: "Router", rules: object, *, log_skipped_rules: bool) -> None:
checked_rules: Final = _checked_content_policy_rules(rules)
runtime_rules: Final = tuple(result for _, result in checked_rules if not isinstance(result, ValidationError))
layered_rules: Final = _layered_over_config_rules(
runtime_rules,
_config_file_rules(
litellm.content_policy_fallbacks # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # untyped module list
),
)
for rule, result in checked_rules:
if log_skipped_rules and isinstance(result, ValidationError):
_log_ignored_content_policy_rules(rule, result)
if layered_rules == llm_router.settings().content_policy_fallbacks: # pyright: ignore[reportUnknownMemberType] # untyped RouterSettings list
return
llm_router.apply(ChangeSet(settings={"content_policy_fallbacks": layered_rules}))
def _apply_fallback_rules(llm_router: "Router", fallback_key: str, rules: Sequence[StoredFallback]) -> None:
if fallback_key == "content_policy_fallbacks":
apply_content_policy_fallbacks(llm_router, rules, log_skipped_rules=True)
return
llm_router.apply(ChangeSet(assign={fallback_key: rules}))
async def _reload_config_file_ownership(proxy_config: "ProxyConfig") -> None:
await proxy_config.get_config()
@ -83,6 +155,7 @@ def _editable_rules(
async def _write_rules(
prisma_client: PrismaClient,
proxy_config: "ProxyConfig",
llm_router: "Router",
stored_settings: Mapping[str, JsonValue],
fallback_key: str,
rules: Sequence[StoredFallback],
@ -91,8 +164,10 @@ async def _write_rules(
section_name=ROUTER_SETTINGS_PARAM, changed_keys={fallback_key: _JSON_RULES.validate_python(rules)}
)
if proxy_config.router_settings.owned_by_config(fallback_key):
llm_router.apply(ChangeSet(assign={fallback_key: rules}))
return
await _persist_router_settings(prisma_client, {**stored_settings, fallback_key: rules})
_apply_fallback_rules(llm_router, fallback_key, rules)
async def _persist_router_settings(prisma_client: PrismaClient, router_settings: Mapping[str, object]) -> None:
@ -218,10 +293,7 @@ async def create_fallback(
# Add new fallback
existing_fallbacks.append({data.model: data.fallback_models})
await _write_rules(prisma_client, proxy_config, router_settings, fallback_key, existing_fallbacks)
# Update the in-memory router configuration
llm_router.apply(ChangeSet(assign={fallback_key: existing_fallbacks}))
await _write_rules(prisma_client, proxy_config, llm_router, router_settings, fallback_key, existing_fallbacks)
verbose_proxy_logger.info(
"Fallback configured: %s -> %s (type: %s)", data.model, data.fallback_models, data.fallback_type
@ -277,7 +349,12 @@ async def get_fallback(
)
# Get fallbacks using the existing utility function
fallback_models: Final = get_all_fallbacks(model=model, llm_router=llm_router, fallback_type=fallback_type)
fallback_targets: Final = _FALLBACK_TARGETS.validate_python(
get_all_fallbacks(model=model, llm_router=llm_router, fallback_type=fallback_type)
)
fallback_models: Final = [
name for target in fallback_targets if (name := fallback_target_model_name(target)) is not None
]
if not fallback_models:
raise HTTPException(
@ -367,10 +444,7 @@ async def delete_fallback(
detail={"error": f"No {fallback_type} fallbacks configured for model '{model}'"},
)
await _write_rules(prisma_client, proxy_config, router_settings, fallback_key, updated_fallbacks)
# Update the in-memory router configuration
llm_router.apply(ChangeSet(assign={fallback_key: updated_fallbacks}))
await _write_rules(prisma_client, proxy_config, llm_router, router_settings, fallback_key, updated_fallbacks)
verbose_proxy_logger.info("Fallback deleted: %s (type: %s)", model, fallback_type)

View file

@ -717,6 +717,9 @@ from litellm.proxy.management_endpoints.customer_endpoints import (
router as customer_router,
)
from litellm.proxy.management_endpoints.daily_activity_routes import router as daily_activity_router
from litellm.proxy.management_endpoints.fallback_management_endpoints import (
apply_content_policy_fallbacks,
)
from litellm.proxy.management_endpoints.fallback_management_endpoints import (
router as fallback_management_router,
)
@ -8159,14 +8162,38 @@ class ProxyConfig:
if db_router_settings is not None and db_router_settings.param_value is not None
else _EMPTY_SETTINGS_MAPPING
)
previous_content_policy_rules: Final = self.router_settings.resolved().get("content_policy_fallbacks")
self.router_settings.apply_db_row("router_settings", db_values)
combined_router_settings: Final = self.router_settings.resolved()
if combined_router_settings:
self._apply_router_settings(llm_router, combined_router_settings)
self._apply_router_settings(
llm_router,
combined_router_settings,
content_policy_owned_by_config=self.router_settings.owned_by_config("content_policy_fallbacks"),
content_policy_rules_changed=(
combined_router_settings.get("content_policy_fallbacks") != previous_content_policy_rules
),
)
@staticmethod
def _apply_router_settings(llm_router: Router, router_settings: Mapping[str, object]) -> None:
llm_router.apply(ChangeSet(settings={k: v for k, v in router_settings.items() if k != "routing_groups"}))
def _apply_router_settings(
llm_router: Router,
router_settings: Mapping[str, object],
*,
content_policy_owned_by_config: bool,
content_policy_rules_changed: bool,
) -> None:
llm_router.apply(
ChangeSet(
settings={
k: v for k, v in router_settings.items() if k not in ("routing_groups", "content_policy_fallbacks")
}
)
)
if "content_policy_fallbacks" in router_settings and not content_policy_owned_by_config:
apply_content_policy_fallbacks(
llm_router, router_settings["content_policy_fallbacks"], log_skipped_rules=content_policy_rules_changed
)
if "routing_groups" not in router_settings:
return
try:

View file

@ -192,6 +192,11 @@ OptionalPreCallChecks = list[
]
FallbackTargets: TypeAlias = Sequence[str | Mapping[str, JsonValue]]
ContentPolicyFallbackRule: TypeAlias = Annotated[Mapping[str, FallbackTargets], Field(min_length=1, max_length=1)]
ContentPolicyFallbackRules: TypeAlias = Sequence[ContentPolicyFallbackRule]
class UpdateRouterConfig(LiteLLMBaseModel):
"""
Set of params that you can modify via `router.update_settings()`.
@ -211,6 +216,7 @@ class UpdateRouterConfig(LiteLLMBaseModel):
retry_after: float | None = None
fallbacks: list[dict] | None = None
context_window_fallbacks: list[dict] | None = None
content_policy_fallbacks: ContentPolicyFallbackRules | None = None
model_group_alias: dict[str, str | dict] | None = Field(default={})
enable_tag_filtering: bool | None = None
weights: RouterWeights | None = None

View file

@ -7597,6 +7597,7 @@ async def test_generate_key_with_router_settings(monkeypatch):
"num_retries": 3,
"model_group_retry_policy": {"gpt-4": {"RateLimitErrorRetries": 5}},
"weights": {"gpt-4": {"weighted-id": 1}},
"content_policy_fallbacks": [{"gpt-4": ["gpt-4o", {"model": "gpt-4o-mini", "temperature": 0}]}],
}
request_data = GenerateKeyRequest(

View file

@ -12,7 +12,7 @@ import copy
import json
from collections.abc import AsyncIterator, Mapping
from types import MappingProxyType, SimpleNamespace
from typing import Final, get_args
from typing import Final, Literal, get_args
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
@ -20,6 +20,7 @@ from fastapi import HTTPException
from pydantic import JsonValue, TypeAdapter
from redis.exceptions import NoPermissionError
import litellm
from litellm import Router
from litellm.proxy.config_resolvers.settings_rules import Section
from litellm.proxy.management_endpoints.fallback_management_endpoints import (
@ -388,6 +389,74 @@ class TestFallbackWritesSeeTheLatestStoredRules:
assert stored.stored_fallbacks() == [GATEWAY_RULE, TEAM_RULE]
async def test_content_policy_writes_keep_the_litellm_settings_rules_for_other_models(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "content_policy_fallbacks", [TEAM_RULE])
router: Final = _team_router()
stored: Final = _StoredRouterSettings(None)
with (
patch("litellm.proxy.proxy_server.llm_router", router),
patch("litellm.proxy.proxy_server.prisma_client", stored.prisma_client),
patch("litellm.proxy.proxy_server.proxy_config", stored.proxy_config),
patch("litellm.proxy.proxy_server.store_model_in_db", True),
):
await create_fallback(
FallbackCreateRequest(
model="gpt-5.4-mini", fallback_models=["team-fallback"], fallback_type="content_policy"
),
MagicMock(),
)
after_create: Final = await get_fallback("team-primary", "content_policy", MagicMock())
await delete_fallback("gpt-5.4-mini", "content_policy", MagicMock())
after_delete: Final = await get_fallback("team-primary", "content_policy", MagicMock())
assert after_create.fallback_models == ["team-fallback"]
assert after_delete.fallback_models == ["team-fallback"]
async def test_a_content_policy_write_applies_beside_an_invalid_stored_rule(self) -> None:
router: Final = _team_router()
stored: Final = _StoredRouterSettings({"content_policy_fallbacks": [MALFORMED_GATEWAY_RULE]})
with (
patch("litellm.proxy.proxy_server.llm_router", router),
patch("litellm.proxy.proxy_server.prisma_client", stored.prisma_client),
patch("litellm.proxy.proxy_server.proxy_config", stored.proxy_config),
patch("litellm.proxy.proxy_server.store_model_in_db", True),
):
await create_fallback(
FallbackCreateRequest(
model="team-primary", fallback_models=["team-fallback"], fallback_type="content_policy"
),
MagicMock(),
)
after_create: Final = await get_fallback("team-primary", "content_policy", MagicMock())
assert after_create.fallback_models == ["team-fallback"]
@pytest.mark.parametrize(
("fallback_type", "router_setting"),
[("general", "fallbacks"), ("content_policy", "content_policy_fallbacks")],
)
async def test_get_names_the_model_of_a_dict_target(
self, fallback_type: Literal["general", "content_policy"], router_setting: str
) -> None:
router: Final = _team_router()
router.update_settings(
**{router_setting: [{"team-primary": [{"model": "gpt-5.4-mini", "api_key": "sk-test"}, "team-fallback"]}]}
)
with patch("litellm.proxy.proxy_server.llm_router", router):
response: Final = await get_fallback("team-primary", fallback_type, MagicMock())
assert response.fallback_models == ["gpt-5.4-mini", "team-fallback"]
async def test_get_answers_an_error_for_a_string_valued_rule_instead_of_its_characters(self) -> None:
router: Final = _team_router()
router.update_settings(fallbacks=[{"team-primary": "team-fallback"}])
with patch("litellm.proxy.proxy_server.llm_router", router), pytest.raises(HTTPException) as raised:
await get_fallback("team-primary", "general", MagicMock())
assert raised.value.status_code == 500
async def test_a_create_keeps_a_rule_another_instance_stored_since_this_one_last_read(self) -> None:
stored: Final = _StoredRouterSettings({})
await get_config_param(stored.prisma_client, "router_settings")
@ -568,6 +637,21 @@ class TestFallbackWritesHonourTheConfigFile:
assert stored.row is None
assert router.fallbacks == [CONFIG_RULE, CONFIG_WINDOW_RULE]
async def test_a_content_policy_restate_keeps_the_config_list_without_the_litellm_settings_rules(
self, router: Router, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "content_policy_fallbacks", [TEAM_RULE])
stored: Final = _StoredRouterSettings(None, config_file={"content_policy_fallbacks": [CONFIG_RULE]})
request: Final = FallbackCreateRequest(
model="gpt-5.4-mini", fallback_models=["team-fallback"], fallback_type="content_policy"
)
response: Final = await self._create(request, router, stored)
assert response.fallback_models == ["team-fallback"]
assert stored.row is None
assert router.content_policy_fallbacks == [CONFIG_RULE]
async def test_a_create_that_restates_a_config_rule_next_to_an_env_reference_writes_nothing(
self, router: Router, monkeypatch: pytest.MonkeyPatch
) -> None:

View file

@ -10087,9 +10087,12 @@ class TestResolveUserProviderCredentials:
assert data["secret_fields"]["user_provider_credentials_user_id"] == "user-a"
@pytest.mark.asyncio
async def test_key_router_settings_fallback_loads_per_user_tokens(self, monkeypatch):
"""The proxy prefers key/team router_settings.fallbacks over router-level
fallbacks (_configured_fallbacks), so discovery must read them too."""
@pytest.mark.parametrize("fallback_key", ["fallbacks", "context_window_fallbacks", "content_policy_fallbacks"])
async def test_key_router_settings_fallback_loads_per_user_tokens(
self, monkeypatch: pytest.MonkeyPatch, fallback_key: str
):
"""The proxy forwards key/team router_settings fallback lists into the
request after discovery, so discovery must read every one of them."""
from litellm.proxy.litellm_pre_call_utils import _resolve_user_provider_credentials_for_request
table = self._env(monkeypatch)
@ -10119,7 +10122,7 @@ class TestResolveUserProviderCredentials:
authenticated_user_id="user-a",
team_id=None,
llm_router=router,
router_settings={"fallbacks": [{"gpt-4o": [{"model": "copilot-chat"}]}]},
router_settings={fallback_key: [{"gpt-4o": [{"model": "copilot-chat"}]}]},
)
table.find_many.assert_awaited()
assert data["secret_fields"]["user_provider_credentials_user_id"] == "user-a"

View file

@ -10,6 +10,7 @@ import subprocess
import sys
import time
import types
from collections.abc import Callable
from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Final
@ -5583,6 +5584,200 @@ async def test_valid_db_routing_groups_still_replace_router_groups():
assert router._get_routing_context("m2", None)[0] == "least-busy"
def _content_policy_router(**router_kwargs: object) -> litellm.Router:
return litellm.Router(
model_list=[
{
"model_name": "primary",
"litellm_params": {"model": "openai/gpt-5.4-mini", "api_key": "fake", "mock_response": "primary answered"},
},
{
"model_name": "secondary",
"litellm_params": {"model": "openai/gpt-5.4-mini", "api_key": "fake", "mock_response": "secondary answered"},
},
{
"model_name": "backup",
"litellm_params": {"model": "openai/gpt-5.4-nano", "api_key": "fake", "mock_response": "backup answered"},
},
],
num_retries=0,
**router_kwargs,
)
def _router_settings_row(content_policy_fallbacks: object) -> MagicMock:
row: Final = MagicMock()
row.param_value = {"allowed_fails": 7, "content_policy_fallbacks": content_policy_fallbacks}
prisma: Final = MagicMock()
prisma.db.litellm_config.find_first = AsyncMock(return_value=row)
return prisma
async def _answer_after_content_policy_refusal(router: litellm.Router, model: str = "primary") -> str | None:
response: Final = await router.acompletion(
model=model, messages=[{"role": "user", "content": "hi"}], mock_testing_content_policy_fallbacks=True
)
return response.choices[0].message.content
@pytest.mark.asyncio
@pytest.mark.parametrize("stored_target", ["backup", {"model": "backup"}])
async def test_db_content_policy_fallbacks_reach_the_router(stored_target: str | dict[str, str]):
router: Final = _content_policy_router()
await proxy_server_module.ProxyConfig()._add_router_settings_from_db_config(
llm_router=router, prisma_client=_router_settings_row([{"primary": [stored_target]}])
)
assert await _answer_after_content_policy_refusal(router) == "backup answered"
@pytest.mark.asyncio
@pytest.mark.parametrize(
("litellm_settings_rules", "stored_rules", "refused_model"),
[
([{"secondary": ["backup"]}], [{"primary": ["backup"]}], "secondary"),
([{"*": ["secondary"]}], [{"*": ["backup"]}], "primary"),
([{"primary": ["backup"]}], [], "primary"),
],
)
async def test_db_content_policy_fallbacks_layer_over_the_litellm_settings_rules(
monkeypatch: pytest.MonkeyPatch,
litellm_settings_rules: list[dict[str, list[str]]],
stored_rules: list[dict[str, list[str]]],
refused_model: str,
):
monkeypatch.setattr(litellm, "content_policy_fallbacks", litellm_settings_rules)
router: Final = _content_policy_router()
await proxy_server_module.ProxyConfig()._add_router_settings_from_db_config(
llm_router=router, prisma_client=_router_settings_row(stored_rules)
)
assert await _answer_after_content_policy_refusal(router, refused_model) == "backup answered"
@pytest.mark.asyncio
@pytest.mark.parametrize(
"invalid_rule",
[{"secondary": "backup"}, ["backup"], dict[str, list[str]](), {"secondary": ["backup"], "primary": ["secondary"]}],
)
async def test_an_invalid_db_content_policy_rule_skips_only_itself(monkeypatch: pytest.MonkeyPatch, invalid_rule: object):
monkeypatch.setattr(litellm, "content_policy_fallbacks", [{"primary": ["backup"]}])
router: Final = _content_policy_router()
await proxy_server_module.ProxyConfig()._add_router_settings_from_db_config(
llm_router=router, prisma_client=_router_settings_row([invalid_rule, {"secondary": ["backup"]}])
)
assert router.allowed_fails == 7
assert await _answer_after_content_policy_refusal(router) == "backup answered"
assert await _answer_after_content_policy_refusal(router, "secondary") == "backup answered"
@pytest.mark.asyncio
async def test_a_db_content_policy_value_that_is_not_a_list_keeps_the_litellm_settings_rules(
monkeypatch: pytest.MonkeyPatch,
):
monkeypatch.setattr(litellm, "content_policy_fallbacks", [{"primary": ["backup"]}])
router: Final = _content_policy_router()
await proxy_server_module.ProxyConfig()._add_router_settings_from_db_config(
llm_router=router, prisma_client=_router_settings_row("oops")
)
assert router.allowed_fails == 7
assert await _answer_after_content_policy_refusal(router) == "backup answered"
@pytest.mark.asyncio
async def test_a_db_sync_keeps_a_litellm_settings_rule_with_a_target_router_init_accepted(
monkeypatch: pytest.MonkeyPatch,
):
monkeypatch.setattr(litellm, "content_policy_fallbacks", [{"primary": ["backup", 5]}])
router: Final = _content_policy_router()
await proxy_server_module.ProxyConfig()._add_router_settings_from_db_config(
llm_router=router, prisma_client=_router_settings_row([{"secondary": ["backup"]}])
)
assert await _answer_after_content_policy_refusal(router) == "backup answered"
assert await _answer_after_content_policy_refusal(router, "secondary") == "backup answered"
@pytest.mark.asyncio
async def test_an_invalid_db_content_policy_rule_is_logged_once_per_change_of_the_rules(
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
):
monkeypatch.setattr(litellm, "content_policy_fallbacks", [])
router: Final = _content_policy_router()
proxy_config: Final = proxy_server_module.ProxyConfig()
stored_rules: Final = [{"secondary": "backup"}, {"primary": ["backup"]}]
changed_rules: Final = [{"secondary": "backup"}, {"secondary": ["backup"]}]
with caplog.at_level(logging.ERROR, logger="LiteLLM Proxy"):
await proxy_config._add_router_settings_from_db_config(
llm_router=router, prisma_client=_router_settings_row(stored_rules)
)
await proxy_config._add_router_settings_from_db_config(
llm_router=router, prisma_client=_router_settings_row(stored_rules)
)
await proxy_config._add_router_settings_from_db_config(
llm_router=router, prisma_client=_router_settings_row(changed_rules)
)
notices: Final = [
record for record in caplog.records if "Ignoring invalid content_policy_fallbacks" in record.getMessage()
]
assert len(notices) == 2
assert await _answer_after_content_policy_refusal(router, "secondary") == "backup answered"
@pytest.mark.asyncio
async def test_a_db_sync_logs_a_new_invalid_content_policy_rule_when_no_valid_rule_changes(
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
):
monkeypatch.setattr(litellm, "content_policy_fallbacks", [])
router: Final = _content_policy_router()
proxy_config: Final = proxy_server_module.ProxyConfig()
with caplog.at_level(logging.ERROR, logger="LiteLLM Proxy"):
await proxy_config._add_router_settings_from_db_config(
llm_router=router, prisma_client=_router_settings_row([{"secondary": "backup"}])
)
await proxy_config._add_router_settings_from_db_config(
llm_router=router, prisma_client=_router_settings_row([{"secondary": "backup"}, {"primary": "backup"}])
)
notices: Final = [
record.getMessage()
for record in caplog.records
if "Ignoring invalid content_policy_fallbacks" in record.getMessage()
]
assert len(notices) == 3
assert "{'primary': 'backup'}" in notices[-1]
@pytest.mark.asyncio
async def test_config_file_content_policy_fallbacks_stay_the_only_rules_after_a_db_sync(
monkeypatch: pytest.MonkeyPatch,
):
monkeypatch.setattr(litellm, "content_policy_fallbacks", [{"secondary": ["backup"]}])
file_rules: Final = [{"primary": ["backup"]}]
router: Final = _content_policy_router(content_policy_fallbacks=file_rules)
proxy_config: Final = proxy_server_module.ProxyConfig()
proxy_config.router_settings.load_yaml({"content_policy_fallbacks": file_rules})
await proxy_config._add_router_settings_from_db_config(
llm_router=router, prisma_client=_router_settings_row([{"secondary": ["backup"]}])
)
assert router.allowed_fails == 7
assert await _answer_after_content_policy_refusal(router) == "backup answered"
with pytest.raises(litellm.ContentPolicyViolationError):
await _answer_after_content_policy_refusal(router, "secondary")
@pytest.mark.asyncio
async def test_add_router_settings_from_db_config_empty_db_lists_do_not_clobber_config_fallbacks():
"""
@ -5591,6 +5786,7 @@ async def test_add_router_settings_from_db_config_empty_db_lists_do_not_clobber_
empty lists are "no value" and must not clobber config.yaml fallbacks,
matching _deep_merge_dicts semantics. Non-empty DB lists still win.
"""
from collections import ChainMap
from unittest.mock import AsyncMock, MagicMock
from litellm.proxy.proxy_server import ProxyConfig
@ -5625,10 +5821,10 @@ async def test_add_router_settings_from_db_config_empty_db_lists_do_not_clobber_
prisma_client=mock_prisma_client,
)
combined_settings = mock_router.update_settings.call_args.kwargs
combined_settings = ChainMap(*(call.kwargs for call in reversed(mock_router.update_settings.call_args_list)))
assert combined_settings["fallbacks"] == [{"gpt-oss-120b": ["granite-4-h-small"]}]
assert combined_settings["context_window_fallbacks"] == [{"gpt-oss-120b": ["granite-4-h-small"]}]
assert combined_settings["content_policy_fallbacks"] == [{"gpt-oss-120b": ["granite-4-h-small"]}]
assert combined_settings.get("content_policy_fallbacks") != [{"gpt-oss-120b": ["other-model"]}]
assert combined_settings["num_retries"] == 3
@ -9997,6 +10193,34 @@ def test_update_config_accepts_disjoint_routing_groups(_update_config_setup):
restore()
def test_update_config_validates_content_policy_fallbacks_like_fallbacks(
_update_config_setup: Callable[..., tuple[TestClient, _FakePrismaClient, Callable[[], None]]],
):
client, prisma, restore = _update_config_setup(initial_rows={"router_settings": {"num_retries": 2}})
rule: Final = [{"gpt-5.4-mini": ["gpt-5.4-nano", {"model": "gpt-5.4-nano", "temperature": 1}]}]
malformed: Final = (
"oops",
5,
[1, 2],
[{"gpt-5.4-mini": "gpt-5.4-nano"}],
[{"gpt-5.4-mini": ["gpt-5.4-nano", 7]}],
[dict[str, list[str]]()],
[{"gpt-5.4-mini": ["gpt-5.4-nano"], "gpt-5.4": ["gpt-5.4-nano"]}],
)
try:
rejected: Final = tuple(
client.post("/config/update", json={"router_settings": {"content_policy_fallbacks": value}}).status_code
for value in malformed
)
assert rejected == (422,) * len(malformed)
assert prisma.db.litellm_config.rows["router_settings"] == {"num_retries": 2}
accepted: Final = client.post("/config/update", json={"router_settings": {"content_policy_fallbacks": rule}})
assert accepted.status_code == 200
assert prisma.db.litellm_config.rows["router_settings"] == {"num_retries": 2, "content_policy_fallbacks": rule}
finally:
restore()
def test_update_config_env_var_round_trip_not_double_encrypted(_update_config_setup, monkeypatch):
"""Endpoint-level regression for the /config/update double-encryption bug.

View file

@ -2730,6 +2730,39 @@ def test_update_settings_model_group_alias_drops_cached_group_info():
assert after.input_cost_per_token is not None and after.input_cost_per_token > 0
@pytest.mark.parametrize(
"fallback_kind, mock_testing_param",
[
("fallbacks", "mock_testing_fallbacks"),
("context_window_fallbacks", "mock_testing_context_fallbacks"),
("content_policy_fallbacks", "mock_testing_content_policy_fallbacks"),
],
)
async def test_update_settings_fallback_rule_of_every_kind_is_used_on_its_error(
fallback_kind: str, mock_testing_param: str
):
router: Final = Router(
model_list=[
{
"model_name": "primary",
"litellm_params": {"model": "gpt-5.4-mini", "api_key": "fake", "mock_response": "primary answered"},
},
{
"model_name": "backup",
"litellm_params": {"model": "gpt-5.4-nano", "api_key": "fake", "mock_response": "backup answered"},
},
],
num_retries=0,
)
router.update_settings(**{fallback_kind: [{"primary": ["backup"]}]})
response: Final = await router.acompletion(
model="primary", messages=[{"role": "user", "content": "hi"}], **{mock_testing_param: True}
)
assert response.choices[0].message.content == "backup answered"
_PAID_INPUT_COST_PER_TOKEN: Final = 3e-06
_PAID_OUTPUT_COST_PER_TOKEN: Final = 1.5e-05

View file

@ -48834,6 +48834,12 @@ export interface components {
UpdateRouterConfig: {
/** Allowed Fails */
allowed_fails?: number | null;
/** Content Policy Fallbacks */
content_policy_fallbacks?: {
[key: string]: (string | {
[key: string]: components["schemas"]["JsonValue"];
})[];
}[] | null;
/** Context Window Fallbacks */
context_window_fallbacks?: {
[key: string]: unknown;