From 14d0136f4e278307891ea497be3bac3070dd3819 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 8 Oct 2026 13:03:54 -0700 Subject: [PATCH 1/9] fix(router): apply content_policy_fallbacks when router settings update at runtime --- litellm/constants.py | 1 + tests/unit/test_router/test_router.py | 33 +++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/litellm/constants.py b/litellm/constants.py index c4adc0de22f..6c0111c07c2 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -24,6 +24,7 @@ RUNTIME_UPDATABLE_ROUTER_SETTINGS: Final[frozenset[str]] = frozenset( "retry_after", "fallbacks", "context_window_fallbacks", + "content_policy_fallbacks", "retry_policy", "model_group_retry_policy", "model_group_alias", diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index cd490e127cf..d4166231579 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -2361,6 +2361,39 @@ def test_update_settings_model_group_alias_drops_cached_group_info(): assert after.input_cost_per_token is not None and after.input_cost_per_token > 0 +@pytest.mark.parametrize( + "fallback_kind, mock_testing_param", + [ + ("fallbacks", "mock_testing_fallbacks"), + ("context_window_fallbacks", "mock_testing_context_fallbacks"), + ("content_policy_fallbacks", "mock_testing_content_policy_fallbacks"), + ], +) +async def test_update_settings_fallback_rule_of_every_kind_is_used_on_its_error( + fallback_kind: str, mock_testing_param: str +): + router = Router( + model_list=[ + { + "model_name": "primary", + "litellm_params": {"model": "gpt-5.4-mini", "api_key": "fake", "mock_response": "primary answered"}, + }, + { + "model_name": "backup", + "litellm_params": {"model": "gpt-5.4-nano", "api_key": "fake", "mock_response": "backup answered"}, + }, + ], + num_retries=0, + ) + + router.update_settings(**{fallback_kind: [{"primary": ["backup"]}]}) + + response = await router.acompletion( + model="primary", messages=[{"role": "user", "content": "hi"}], **{mock_testing_param: True} + ) + assert response.choices[0].message.content == "backup answered" + + _PAID_INPUT_COST_PER_TOKEN: Final = 3e-06 _PAID_OUTPUT_COST_PER_TOKEN: Final = 1.5e-05 From 9c1919a3278151051aca9d2b9b5e9b5486eefb5b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 8 Oct 2026 13:45:21 -0700 Subject: [PATCH 2/9] test(router): annotate the fallback regression test's locals as Final --- tests/unit/test_router/test_router.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index d4166231579..c14ddb9e734 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -2372,7 +2372,7 @@ def test_update_settings_model_group_alias_drops_cached_group_info(): async def test_update_settings_fallback_rule_of_every_kind_is_used_on_its_error( fallback_kind: str, mock_testing_param: str ): - router = Router( + router: Final = Router( model_list=[ { "model_name": "primary", @@ -2388,7 +2388,7 @@ async def test_update_settings_fallback_rule_of_every_kind_is_used_on_its_error( router.update_settings(**{fallback_kind: [{"primary": ["backup"]}]}) - response = await router.acompletion( + response: Final = await router.acompletion( model="primary", messages=[{"role": "user", "content": "hi"}], **{mock_testing_param: True} ) assert response.choices[0].message.content == "backup answered" From 9c88a086a636bbae2e6e58e27544b66c8c986f0f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 8 Oct 2026 17:45:10 -0700 Subject: [PATCH 3/9] fix(proxy): validate router_settings.content_policy_fallbacks on /config/update like its siblings --- litellm/types/router.py | 1 + .../test_proxy_server_endpoints_and_startup.py | 15 +++++++++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 4 ++++ 3 files changed, 20 insertions(+) diff --git a/litellm/types/router.py b/litellm/types/router.py index a66c4571b39..756e36d2fc7 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -181,6 +181,7 @@ class UpdateRouterConfig(LiteLLMBaseModel): retry_after: float | None = None fallbacks: list[dict] | None = None context_window_fallbacks: list[dict] | None = None + content_policy_fallbacks: Sequence[Mapping[str, Sequence[str]]] | None = None model_group_alias: dict[str, str | dict] | None = Field(default={}) enable_tag_filtering: bool | None = None weights: RouterWeights | None = None diff --git a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py index 920f9d90bb8..8c6822f85bb 100644 --- a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py +++ b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py @@ -10115,6 +10115,21 @@ def test_update_config_accepts_disjoint_routing_groups(_update_config_setup): restore() +def test_update_config_validates_content_policy_fallbacks_like_fallbacks(_update_config_setup): + client, prisma, restore = _update_config_setup(initial_rows={"router_settings": {"num_retries": 2}}) + rule: Final = [{"gpt-5.4-mini": ["gpt-5.4-nano"]}] + try: + for malformed in ("oops", 5, [1, 2], [{"gpt-5.4-mini": "gpt-5.4-nano"}], [{"gpt-5.4-mini": ["gpt-5.4-nano", 7]}]): + rejected = client.post("/config/update", json={"router_settings": {"content_policy_fallbacks": malformed}}) + assert rejected.status_code == 422, malformed + assert prisma.db.litellm_config.rows["router_settings"] == {"num_retries": 2} + accepted = client.post("/config/update", json={"router_settings": {"content_policy_fallbacks": rule}}) + assert accepted.status_code == 200 + assert prisma.db.litellm_config.rows["router_settings"] == {"num_retries": 2, "content_policy_fallbacks": rule} + finally: + restore() + + def test_update_config_env_var_round_trip_not_double_encrypted(_update_config_setup, monkeypatch): """Endpoint-level regression for the /config/update double-encryption bug. diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 769798b5134..ccefbd7901b 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -49206,6 +49206,10 @@ export interface components { UpdateRouterConfig: { /** Allowed Fails */ allowed_fails?: number | null; + /** Content Policy Fallbacks */ + content_policy_fallbacks?: { + [key: string]: string[]; + }[] | null; /** Context Window Fallbacks */ context_window_fallbacks?: { [key: string]: unknown; From 5f9aa12b71ed2a8d899d4867a2cd9f954b8cb92e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 8 Oct 2026 18:58:32 -0700 Subject: [PATCH 4/9] test(proxy): annotate the content_policy_fallbacks config update test locals as Final --- .../proxy/test_proxy_server_endpoints_and_startup.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py index 8c6822f85bb..5a6f4c957aa 100644 --- a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py +++ b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py @@ -10118,12 +10118,15 @@ def test_update_config_accepts_disjoint_routing_groups(_update_config_setup): def test_update_config_validates_content_policy_fallbacks_like_fallbacks(_update_config_setup): client, prisma, restore = _update_config_setup(initial_rows={"router_settings": {"num_retries": 2}}) rule: Final = [{"gpt-5.4-mini": ["gpt-5.4-nano"]}] + malformed: Final = ("oops", 5, [1, 2], [{"gpt-5.4-mini": "gpt-5.4-nano"}], [{"gpt-5.4-mini": ["gpt-5.4-nano", 7]}]) try: - for malformed in ("oops", 5, [1, 2], [{"gpt-5.4-mini": "gpt-5.4-nano"}], [{"gpt-5.4-mini": ["gpt-5.4-nano", 7]}]): - rejected = client.post("/config/update", json={"router_settings": {"content_policy_fallbacks": malformed}}) - assert rejected.status_code == 422, malformed + rejected: Final = tuple( + client.post("/config/update", json={"router_settings": {"content_policy_fallbacks": value}}).status_code + for value in malformed + ) + assert rejected == (422,) * len(malformed) assert prisma.db.litellm_config.rows["router_settings"] == {"num_retries": 2} - accepted = client.post("/config/update", json={"router_settings": {"content_policy_fallbacks": rule}}) + accepted: Final = client.post("/config/update", json={"router_settings": {"content_policy_fallbacks": rule}}) assert accepted.status_code == 200 assert prisma.db.litellm_config.rows["router_settings"] == {"num_retries": 2, "content_policy_fallbacks": rule} finally: From 0a1fba1f5a1a3ab9f5e9fa8f0d810cac73fa8a33 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 22:25:14 -0700 Subject: [PATCH 5/9] fix(proxy): keep litellm_settings content policy fallbacks, accept dict targets, skip invalid stored rules A runtime content_policy_fallbacks rule now layers over the litellm_settings rules instead of replacing them, so models the runtime rules do not name keep their file fallbacks. Fallback targets may be deployment dicts as well as model names. A stored rule that fails validation is logged and skipped at sync, so the other router settings still apply. Per-user token discovery now reads every fallback key from key and team router_settings. --- litellm/proxy/litellm_pre_call_utils.py | 17 ++-- .../fallback_management_endpoints.py | 41 +++++++- litellm/proxy/proxy_server.py | 9 +- litellm/types/router.py | 5 +- .../test_key_management_endpoints.py | 1 + .../test_fallback_management_endpoints.py | 26 +++++ .../unit/proxy/test_litellm_pre_call_utils.py | 9 +- ...test_proxy_server_endpoints_and_startup.py | 96 ++++++++++++++++++- ui/litellm-dashboard/src/lib/http/schema.d.ts | 4 +- 9 files changed, 187 insertions(+), 21 deletions(-) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 24a31723faa..08fdb1b3639 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -3139,9 +3139,9 @@ def _fallback_lists( ) -> tuple[Sequence[object], ...]: """Every fallback list the router can route this request through: the three router-level lists, the same three keys from the request body (which replace - the router's when present, so the union is the safe superset), and the - key/team ``router_settings.fallbacks`` the proxy prefers over router-level - fallbacks in ``_configured_fallbacks``.""" + the router's when present, so the union is the safe superset), and the same + three keys from key/team ``router_settings``, which the proxy forwards into + the request after this discovery runs.""" lists: Final[list[Sequence[object]]] = [] # mutable-ok: accumulates each fallback list found for router_list in ( cast("object", llm_router.fallbacks), # cast-ok: router fallback attrs are untyped @@ -3155,11 +3155,12 @@ def _fallback_lists( if isinstance(entries, list): lists.append(cast("Sequence[object]", entries)) # cast-ok: request fallback lists hold mixed entry shapes for settings in (router_settings, team_router_settings): - key_fallbacks: object = settings.get("fallbacks") if settings is not None else None - if isinstance(key_fallbacks, list): - lists.append( - cast("Sequence[object]", key_fallbacks) # cast-ok: settings fallback lists hold mixed entry shapes - ) + for key in _REQUEST_FALLBACK_KEYS: + key_fallbacks: object = settings.get(key) if settings is not None else None + if isinstance(key_fallbacks, list): + lists.append( + cast("Sequence[object]", key_fallbacks) # cast-ok: settings fallback lists hold mixed entry shapes + ) return tuple(lists) diff --git a/litellm/proxy/management_endpoints/fallback_management_endpoints.py b/litellm/proxy/management_endpoints/fallback_management_endpoints.py index 6a9a30a7689..4518bca79bd 100644 --- a/litellm/proxy/management_endpoints/fallback_management_endpoints.py +++ b/litellm/proxy/management_endpoints/fallback_management_endpoints.py @@ -14,8 +14,9 @@ import json from collections.abc import Mapping, Sequence from typing import TYPE_CHECKING, Annotated, Final, Literal -from pydantic import Field, JsonValue, TypeAdapter +from pydantic import Field, JsonValue, TypeAdapter, ValidationError +import litellm from litellm._logging import verbose_proxy_logger from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.model_checks import get_all_fallbacks @@ -26,6 +27,7 @@ if TYPE_CHECKING: from fastapi import APIRouter, Depends, HTTPException, status from litellm.proxy.proxy_server import ProxyConfig + from litellm.router import Router else: try: from fastapi import APIRouter, Depends, HTTPException, status @@ -40,6 +42,7 @@ from litellm.types.management_endpoints.router_settings_endpoints import ( FallbackGetResponse, FallbackResponse, ) +from litellm.types.router import ContentPolicyFallbackRules router: Final = APIRouter() @@ -48,12 +51,44 @@ FallbackRule = dict[str, list[str]] StoredFallback = Annotated[FallbackRule | dict[str, object] | str, Field(union_mode="left_to_right")] _STORED_FALLBACKS: Final = TypeAdapter(list[StoredFallback]) _JSON_RULES: Final = TypeAdapter(list[JsonValue]) +_CONTENT_POLICY_FALLBACK_RULES: Final = TypeAdapter(ContentPolicyFallbackRules) def _rule_covers(entry: StoredFallback, model: str) -> bool: return isinstance(entry, dict) and model in entry +def _layered_over_config_rules( + runtime_rules: ContentPolicyFallbackRules, config_rules: ContentPolicyFallbackRules +) -> ContentPolicyFallbackRules: + covered: Final = frozenset[str]().union(*runtime_rules) + return [*runtime_rules, *(rule for rule in config_rules if covered.isdisjoint(rule))] + + +def apply_content_policy_fallbacks(llm_router: "Router", rules: object) -> None: + try: + runtime_rules: Final = _CONTENT_POLICY_FALLBACK_RULES.validate_python(rules) + config_rules: Final = _CONTENT_POLICY_FALLBACK_RULES.validate_python( + litellm.content_policy_fallbacks or [] # pyright: ignore[reportUnknownMemberType] # untyped module list + ) + except ValidationError as invalid_rules: + verbose_proxy_logger.error( + "Ignoring invalid content_policy_fallbacks from config/DB, all other router settings still apply. " + "Fix the content policy fallbacks in the Admin UI or config to load them: %s", + invalid_rules, + ) + return + layered_rules: Final = _layered_over_config_rules(runtime_rules, config_rules) + llm_router.update_settings(content_policy_fallbacks=layered_rules) # pyright: ignore[reportUnknownMemberType] # untyped kwargs + + +def _apply_fallback_rules(llm_router: "Router", fallback_key: str, rules: Sequence[StoredFallback]) -> None: + if fallback_key == "content_policy_fallbacks": + apply_content_policy_fallbacks(llm_router, rules) + return + setattr(llm_router, fallback_key, rules) + + async def _router_settings_fresh_from_db(proxy_config: "ProxyConfig") -> dict[str, object]: await evict_config_param(ROUTER_SETTINGS_PARAM) config: Final = await proxy_config.get_config() @@ -195,7 +230,7 @@ async def create_fallback( await _persist_router_settings(prisma_client, router_settings) # Update the in-memory router configuration - setattr(llm_router, fallback_key, existing_fallbacks) + _apply_fallback_rules(llm_router, fallback_key, existing_fallbacks) verbose_proxy_logger.info( "Fallback configured: %s -> %s (type: %s)", data.model, data.fallback_models, data.fallback_type @@ -348,7 +383,7 @@ async def delete_fallback( await _persist_router_settings(prisma_client, router_settings) # Update the in-memory router configuration - setattr(llm_router, fallback_key, updated_fallbacks) + _apply_fallback_rules(llm_router, fallback_key, updated_fallbacks) verbose_proxy_logger.info("Fallback deleted: %s (type: %s)", model, fallback_type) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 64fb2791913..d1f1233c905 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -702,6 +702,9 @@ from litellm.proxy.management_endpoints.customer_endpoints import ( router as customer_router, ) from litellm.proxy.management_endpoints.daily_activity_routes import router as daily_activity_router +from litellm.proxy.management_endpoints.fallback_management_endpoints import ( + apply_content_policy_fallbacks, +) from litellm.proxy.management_endpoints.fallback_management_endpoints import ( router as fallback_management_router, ) @@ -8094,7 +8097,11 @@ class ProxyConfig: @staticmethod def _apply_router_settings(llm_router: Router, router_settings: Mapping[str, object]) -> None: - llm_router.update_settings(**{k: v for k, v in router_settings.items() if k != "routing_groups"}) + llm_router.update_settings( + **{k: v for k, v in router_settings.items() if k not in ("routing_groups", "content_policy_fallbacks")} + ) + if "content_policy_fallbacks" in router_settings: + apply_content_policy_fallbacks(llm_router, router_settings["content_policy_fallbacks"]) if "routing_groups" not in router_settings: return try: diff --git a/litellm/types/router.py b/litellm/types/router.py index 06650ef22b3..69b9f8a5ca2 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -185,6 +185,9 @@ OptionalPreCallChecks = list[ ] +ContentPolicyFallbackRules: TypeAlias = Sequence[Mapping[str, Sequence[str | Mapping[str, JsonValue]]]] + + class UpdateRouterConfig(LiteLLMBaseModel): """ Set of params that you can modify via `router.update_settings()`. @@ -204,7 +207,7 @@ class UpdateRouterConfig(LiteLLMBaseModel): retry_after: float | None = None fallbacks: list[dict] | None = None context_window_fallbacks: list[dict] | None = None - content_policy_fallbacks: Sequence[Mapping[str, Sequence[str]]] | None = None + content_policy_fallbacks: ContentPolicyFallbackRules | None = None model_group_alias: dict[str, str | dict] | None = Field(default={}) enable_tag_filtering: bool | None = None weights: RouterWeights | None = None diff --git a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py index 9634f1efce8..39aa3acf1ef 100644 --- a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py @@ -7597,6 +7597,7 @@ async def test_generate_key_with_router_settings(monkeypatch): "num_retries": 3, "model_group_retry_policy": {"gpt-4": {"RateLimitErrorRetries": 5}}, "weights": {"gpt-4": {"weighted-id": 1}}, + "content_policy_fallbacks": [{"gpt-4": ["gpt-4o", {"model": "gpt-4o-mini", "temperature": 0}]}], } request_data = GenerateKeyRequest( diff --git a/tests/unit/proxy/test_fallback_management_endpoints.py b/tests/unit/proxy/test_fallback_management_endpoints.py index 530d22f02be..8bf67dc024b 100644 --- a/tests/unit/proxy/test_fallback_management_endpoints.py +++ b/tests/unit/proxy/test_fallback_management_endpoints.py @@ -19,6 +19,7 @@ import pytest from fastapi import HTTPException from pydantic import JsonValue, TypeAdapter +import litellm from litellm import Router from litellm.proxy.management_endpoints.fallback_management_endpoints import ( FallbackCreateRequest, @@ -331,6 +332,31 @@ class TestFallbackWritesSeeTheLatestStoredRules: assert stored.stored_fallbacks() == [TEAM_RULE, GATEWAY_RULE] assert await stored.cached_fallbacks() == [TEAM_RULE, GATEWAY_RULE] + async def test_content_policy_writes_keep_the_litellm_settings_rules_for_other_models( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setattr(litellm, "content_policy_fallbacks", [TEAM_RULE]) + router: Final = _team_router() + stored: Final = _StoredRouterSettings(None) + with ( + patch("litellm.proxy.proxy_server.llm_router", router), + patch("litellm.proxy.proxy_server.prisma_client", stored.prisma_client), + patch("litellm.proxy.proxy_server.proxy_config", stored.proxy_config), + patch("litellm.proxy.proxy_server.store_model_in_db", True), + ): + await create_fallback( + FallbackCreateRequest( + model="gpt-5.4-mini", fallback_models=["team-fallback"], fallback_type="content_policy" + ), + MagicMock(), + ) + after_create: Final = await get_fallback("team-primary", "content_policy", MagicMock()) + await delete_fallback("gpt-5.4-mini", "content_policy", MagicMock()) + after_delete: Final = await get_fallback("team-primary", "content_policy", MagicMock()) + + assert after_create.fallback_models == ["team-fallback"] + assert after_delete.fallback_models == ["team-fallback"] + async def test_a_create_keeps_a_rule_another_instance_stored_since_this_one_last_read(self) -> None: stored: Final = _StoredRouterSettings({}) await get_config_param(stored.prisma_client, "router_settings") diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index 81a196af676..2fa542c9ef7 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -10071,9 +10071,10 @@ class TestResolveUserProviderCredentials: assert data["secret_fields"]["user_provider_credentials_user_id"] == "user-a" @pytest.mark.asyncio - async def test_key_router_settings_fallback_loads_per_user_tokens(self, monkeypatch): - """The proxy prefers key/team router_settings.fallbacks over router-level - fallbacks (_configured_fallbacks), so discovery must read them too.""" + @pytest.mark.parametrize("fallback_key", ["fallbacks", "context_window_fallbacks", "content_policy_fallbacks"]) + async def test_key_router_settings_fallback_loads_per_user_tokens(self, monkeypatch, fallback_key): + """The proxy forwards key/team router_settings fallback lists into the + request after discovery, so discovery must read every one of them.""" from litellm.proxy.litellm_pre_call_utils import _resolve_user_provider_credentials_for_request table = self._env(monkeypatch) @@ -10103,7 +10104,7 @@ class TestResolveUserProviderCredentials: authenticated_user_id="user-a", team_id=None, llm_router=router, - router_settings={"fallbacks": [{"gpt-4o": [{"model": "copilot-chat"}]}]}, + router_settings={fallback_key: [{"gpt-4o": [{"model": "copilot-chat"}]}]}, ) table.find_many.assert_awaited() assert data["secret_fields"]["user_provider_credentials_user_id"] == "user-a" diff --git a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py index a5b55ac50fb..6860b477621 100644 --- a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py +++ b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py @@ -10,6 +10,7 @@ import subprocess import sys import time import types +from collections.abc import Callable from datetime import datetime, timedelta, timezone from pathlib import Path from typing import Final @@ -5581,6 +5582,92 @@ async def test_valid_db_routing_groups_still_replace_router_groups(): assert router._get_routing_context("m2", None)[0] == "least-busy" +def _content_policy_router(**router_kwargs: object) -> litellm.Router: + return litellm.Router( + model_list=[ + { + "model_name": "primary", + "litellm_params": {"model": "openai/gpt-5.4-mini", "api_key": "fake", "mock_response": "primary answered"}, + }, + { + "model_name": "secondary", + "litellm_params": {"model": "openai/gpt-5.4-mini", "api_key": "fake", "mock_response": "secondary answered"}, + }, + { + "model_name": "backup", + "litellm_params": {"model": "openai/gpt-5.4-nano", "api_key": "fake", "mock_response": "backup answered"}, + }, + ], + num_retries=0, + **router_kwargs, + ) + + +def _router_settings_row(content_policy_fallbacks: object) -> MagicMock: + row: Final = MagicMock() + row.param_value = {"allowed_fails": 7, "content_policy_fallbacks": content_policy_fallbacks} + prisma: Final = MagicMock() + prisma.db.litellm_config.find_first = AsyncMock(return_value=row) + return prisma + + +async def _answer_after_content_policy_refusal(router: litellm.Router, model: str = "primary") -> str | None: + response: Final = await router.acompletion( + model=model, messages=[{"role": "user", "content": "hi"}], mock_testing_content_policy_fallbacks=True + ) + return response.choices[0].message.content + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stored_target", ["backup", {"model": "backup"}]) +async def test_db_content_policy_fallbacks_reach_the_router(stored_target: str | dict[str, str]): + router: Final = _content_policy_router() + + await proxy_server_module.ProxyConfig()._add_router_settings_from_db_config( + llm_router=router, prisma_client=_router_settings_row([{"primary": [stored_target]}]) + ) + + assert await _answer_after_content_policy_refusal(router) == "backup answered" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("litellm_settings_rules", "stored_rules", "refused_model"), + [ + ([{"secondary": ["backup"]}], [{"primary": ["backup"]}], "secondary"), + ([{"*": ["secondary"]}], [{"*": ["backup"]}], "primary"), + ([{"primary": ["backup"]}], [], "primary"), + ], +) +async def test_db_content_policy_fallbacks_layer_over_the_litellm_settings_rules( + monkeypatch: pytest.MonkeyPatch, + litellm_settings_rules: list[dict[str, list[str]]], + stored_rules: list[dict[str, list[str]]], + refused_model: str, +): + monkeypatch.setattr(litellm, "content_policy_fallbacks", litellm_settings_rules) + router: Final = _content_policy_router() + + await proxy_server_module.ProxyConfig()._add_router_settings_from_db_config( + llm_router=router, prisma_client=_router_settings_row(stored_rules) + ) + + assert await _answer_after_content_policy_refusal(router, refused_model) == "backup answered" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stored_rules", ["oops", [{"primary": "backup"}], [["backup"]]]) +async def test_invalid_db_content_policy_fallbacks_keep_the_router_rules(stored_rules: object): + router: Final = _content_policy_router(content_policy_fallbacks=[{"primary": ["backup"]}]) + + await proxy_server_module.ProxyConfig()._add_router_settings_from_db_config( + llm_router=router, prisma_client=_router_settings_row(stored_rules) + ) + + assert router.allowed_fails == 7 + assert await _answer_after_content_policy_refusal(router) == "backup answered" + + @pytest.mark.asyncio async def test_add_router_settings_from_db_config_empty_db_lists_do_not_clobber_config_fallbacks(): """ @@ -5589,6 +5676,7 @@ async def test_add_router_settings_from_db_config_empty_db_lists_do_not_clobber_ empty lists are "no value" and must not clobber config.yaml fallbacks, matching _deep_merge_dicts semantics. Non-empty DB lists still win. """ + from collections import ChainMap from unittest.mock import AsyncMock, MagicMock from litellm.proxy.proxy_server import ProxyConfig @@ -5623,7 +5711,7 @@ async def test_add_router_settings_from_db_config_empty_db_lists_do_not_clobber_ prisma_client=mock_prisma_client, ) - combined_settings = mock_router.update_settings.call_args.kwargs + combined_settings = ChainMap(*(call.kwargs for call in reversed(mock_router.update_settings.call_args_list))) assert combined_settings["fallbacks"] == [{"gpt-oss-120b": ["granite-4-h-small"]}] assert combined_settings["context_window_fallbacks"] == [{"gpt-oss-120b": ["granite-4-h-small"]}] assert combined_settings["content_policy_fallbacks"] == [{"gpt-oss-120b": ["granite-4-h-small"]}] @@ -9962,9 +10050,11 @@ def test_update_config_accepts_disjoint_routing_groups(_update_config_setup): restore() -def test_update_config_validates_content_policy_fallbacks_like_fallbacks(_update_config_setup): +def test_update_config_validates_content_policy_fallbacks_like_fallbacks( + _update_config_setup: Callable[..., tuple[TestClient, _FakePrismaClient, Callable[[], None]]], +): client, prisma, restore = _update_config_setup(initial_rows={"router_settings": {"num_retries": 2}}) - rule: Final = [{"gpt-5.4-mini": ["gpt-5.4-nano"]}] + rule: Final = [{"gpt-5.4-mini": ["gpt-5.4-nano", {"model": "gpt-5.4-nano", "temperature": 1}]}] malformed: Final = ("oops", 5, [1, 2], [{"gpt-5.4-mini": "gpt-5.4-nano"}], [{"gpt-5.4-mini": ["gpt-5.4-nano", 7]}]) try: rejected: Final = tuple( diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 060cd1ce788..3bd81cc1ac9 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -48688,7 +48688,9 @@ export interface components { allowed_fails?: number | null; /** Content Policy Fallbacks */ content_policy_fallbacks?: { - [key: string]: string[]; + [key: string]: (string | { + [key: string]: components["schemas"]["JsonValue"]; + })[]; }[] | null; /** Context Window Fallbacks */ context_window_fallbacks?: { From 9f0b89546a2a912f08bc68f8de61bf0f9ca11ab8 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 05:14:55 -0700 Subject: [PATCH 6/9] fix(proxy): validate content policy fallback rules one by one and keep a config-owned list alone An invalid stored rule is now skipped on its own instead of dropping every runtime rule, a router_settings list from the config file stays the router's only rules across DB syncs, and GET /fallback names the model of a dict target. --- .../fallback_management_endpoints.py | 54 ++++++++++++----- litellm/proxy/proxy_server.py | 12 +++- litellm/types/router.py | 5 +- .../test_fallback_management_endpoints.py | 37 +++++++++++- .../unit/proxy/test_litellm_pre_call_utils.py | 4 +- ...test_proxy_server_endpoints_and_startup.py | 60 +++++++++++++++++-- 6 files changed, 145 insertions(+), 27 deletions(-) diff --git a/litellm/proxy/management_endpoints/fallback_management_endpoints.py b/litellm/proxy/management_endpoints/fallback_management_endpoints.py index 4518bca79bd..6850e0d5bdc 100644 --- a/litellm/proxy/management_endpoints/fallback_management_endpoints.py +++ b/litellm/proxy/management_endpoints/fallback_management_endpoints.py @@ -19,6 +19,7 @@ from pydantic import Field, JsonValue, TypeAdapter, ValidationError import litellm from litellm._logging import verbose_proxy_logger from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.auth.auth_utils import fallback_target_model_name from litellm.proxy.auth.model_checks import get_all_fallbacks from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.utils import PrismaClient, evict_config_param, invalidate_config_param @@ -42,7 +43,7 @@ from litellm.types.management_endpoints.router_settings_endpoints import ( FallbackGetResponse, FallbackResponse, ) -from litellm.types.router import ContentPolicyFallbackRules +from litellm.types.router import ContentPolicyFallbackRule, ContentPolicyFallbackRules router: Final = APIRouter() @@ -51,7 +52,8 @@ FallbackRule = dict[str, list[str]] StoredFallback = Annotated[FallbackRule | dict[str, object] | str, Field(union_mode="left_to_right")] _STORED_FALLBACKS: Final = TypeAdapter(list[StoredFallback]) _JSON_RULES: Final = TypeAdapter(list[JsonValue]) -_CONTENT_POLICY_FALLBACK_RULES: Final = TypeAdapter(ContentPolicyFallbackRules) +_OPTIONAL_JSON_RULES: Final[TypeAdapter[Sequence[JsonValue] | None]] = TypeAdapter(Sequence[JsonValue] | None) +_CONTENT_POLICY_FALLBACK_RULE: Final[TypeAdapter[ContentPolicyFallbackRule]] = TypeAdapter(ContentPolicyFallbackRule) def _rule_covers(entry: StoredFallback, model: str) -> bool: @@ -65,19 +67,38 @@ def _layered_over_config_rules( return [*runtime_rules, *(rule for rule in config_rules if covered.isdisjoint(rule))] -def apply_content_policy_fallbacks(llm_router: "Router", rules: object) -> None: +def _log_ignored_content_policy_rules(rules: object, error: ValidationError) -> None: + verbose_proxy_logger.error( + "Ignoring invalid content_policy_fallbacks %r from config/DB, every other rule still applies. " + "Fix it through POST /fallback, /config/update, or the config file: %s", + rules, + error, + ) + + +def _valid_content_policy_rule(rule: JsonValue) -> ContentPolicyFallbackRule | None: try: - runtime_rules: Final = _CONTENT_POLICY_FALLBACK_RULES.validate_python(rules) - config_rules: Final = _CONTENT_POLICY_FALLBACK_RULES.validate_python( - litellm.content_policy_fallbacks or [] # pyright: ignore[reportUnknownMemberType] # untyped module list - ) - except ValidationError as invalid_rules: - verbose_proxy_logger.error( - "Ignoring invalid content_policy_fallbacks from config/DB, all other router settings still apply. " - "Fix the content policy fallbacks in the Admin UI or config to load them: %s", - invalid_rules, - ) - return + return _CONTENT_POLICY_FALLBACK_RULE.validate_python(rule) + except ValidationError as invalid_rule: + _log_ignored_content_policy_rules(rule, invalid_rule) + return None + + +def _valid_content_policy_rules(rules: object) -> ContentPolicyFallbackRules: + try: + candidates: Final = _OPTIONAL_JSON_RULES.validate_python(rules) or () + except ValidationError as not_a_rule_list: + _log_ignored_content_policy_rules(rules, not_a_rule_list) + return () + validated: Final = (_valid_content_policy_rule(candidate) for candidate in candidates) + return tuple(rule for rule in validated if rule is not None) + + +def apply_content_policy_fallbacks(llm_router: "Router", rules: object) -> None: + runtime_rules: Final = _valid_content_policy_rules(rules) + config_rules: Final = _valid_content_policy_rules( + litellm.content_policy_fallbacks # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # untyped module list + ) layered_rules: Final = _layered_over_config_rules(runtime_rules, config_rules) llm_router.update_settings(content_policy_fallbacks=layered_rules) # pyright: ignore[reportUnknownMemberType] # untyped kwargs @@ -286,7 +307,10 @@ async def get_fallback( ) # Get fallbacks using the existing utility function - fallback_models: Final = get_all_fallbacks(model=model, llm_router=llm_router, fallback_type=fallback_type) + fallback_targets: Final = get_all_fallbacks(model=model, llm_router=llm_router, fallback_type=fallback_type) + fallback_models: Final = [ + name for target in fallback_targets if (name := fallback_target_model_name(target)) is not None + ] if not fallback_models: raise HTTPException( diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index d1f1233c905..e999729e27b 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -8093,14 +8093,20 @@ class ProxyConfig: self.router_settings.apply_db_row("router_settings", db_values) combined_router_settings: Final = self.router_settings.resolved() if combined_router_settings: - self._apply_router_settings(llm_router, combined_router_settings) + self._apply_router_settings( + llm_router, + combined_router_settings, + content_policy_owned_by_config=self.router_settings.owned_by_config("content_policy_fallbacks"), + ) @staticmethod - def _apply_router_settings(llm_router: Router, router_settings: Mapping[str, object]) -> None: + def _apply_router_settings( + llm_router: Router, router_settings: Mapping[str, object], *, content_policy_owned_by_config: bool + ) -> None: llm_router.update_settings( **{k: v for k, v in router_settings.items() if k not in ("routing_groups", "content_policy_fallbacks")} ) - if "content_policy_fallbacks" in router_settings: + if "content_policy_fallbacks" in router_settings and not content_policy_owned_by_config: apply_content_policy_fallbacks(llm_router, router_settings["content_policy_fallbacks"]) if "routing_groups" not in router_settings: return diff --git a/litellm/types/router.py b/litellm/types/router.py index 69b9f8a5ca2..e79d14b91cd 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -185,7 +185,10 @@ OptionalPreCallChecks = list[ ] -ContentPolicyFallbackRules: TypeAlias = Sequence[Mapping[str, Sequence[str | Mapping[str, JsonValue]]]] +ContentPolicyFallbackRule: TypeAlias = Annotated[ + Mapping[str, Sequence[str | Mapping[str, JsonValue]]], Field(min_length=1, max_length=1) +] +ContentPolicyFallbackRules: TypeAlias = Sequence[ContentPolicyFallbackRule] class UpdateRouterConfig(LiteLLMBaseModel): diff --git a/tests/unit/proxy/test_fallback_management_endpoints.py b/tests/unit/proxy/test_fallback_management_endpoints.py index 8bf67dc024b..2dc376bb4f3 100644 --- a/tests/unit/proxy/test_fallback_management_endpoints.py +++ b/tests/unit/proxy/test_fallback_management_endpoints.py @@ -12,7 +12,7 @@ import copy import json from collections.abc import AsyncIterator from types import SimpleNamespace -from typing import Final +from typing import Final, Literal from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -357,6 +357,41 @@ class TestFallbackWritesSeeTheLatestStoredRules: assert after_create.fallback_models == ["team-fallback"] assert after_delete.fallback_models == ["team-fallback"] + async def test_a_content_policy_write_applies_beside_an_invalid_stored_rule(self) -> None: + router: Final = _team_router() + stored: Final = _StoredRouterSettings({"content_policy_fallbacks": [MALFORMED_GATEWAY_RULE]}) + with ( + patch("litellm.proxy.proxy_server.llm_router", router), + patch("litellm.proxy.proxy_server.prisma_client", stored.prisma_client), + patch("litellm.proxy.proxy_server.proxy_config", stored.proxy_config), + patch("litellm.proxy.proxy_server.store_model_in_db", True), + ): + await create_fallback( + FallbackCreateRequest( + model="team-primary", fallback_models=["team-fallback"], fallback_type="content_policy" + ), + MagicMock(), + ) + after_create: Final = await get_fallback("team-primary", "content_policy", MagicMock()) + + assert after_create.fallback_models == ["team-fallback"] + + @pytest.mark.parametrize( + ("fallback_type", "router_setting"), + [("general", "fallbacks"), ("content_policy", "content_policy_fallbacks")], + ) + async def test_get_names_the_model_of_a_dict_target( + self, fallback_type: Literal["general", "content_policy"], router_setting: str + ) -> None: + router: Final = _team_router() + router.update_settings( + **{router_setting: [{"team-primary": [{"model": "gpt-5.4-mini", "api_key": "sk-test"}, "team-fallback"]}]} + ) + with patch("litellm.proxy.proxy_server.llm_router", router): + response: Final = await get_fallback("team-primary", fallback_type, MagicMock()) + + assert response.fallback_models == ["gpt-5.4-mini", "team-fallback"] + async def test_a_create_keeps_a_rule_another_instance_stored_since_this_one_last_read(self) -> None: stored: Final = _StoredRouterSettings({}) await get_config_param(stored.prisma_client, "router_settings") diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index 2fa542c9ef7..2697dbce0d6 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -10072,7 +10072,9 @@ class TestResolveUserProviderCredentials: @pytest.mark.asyncio @pytest.mark.parametrize("fallback_key", ["fallbacks", "context_window_fallbacks", "content_policy_fallbacks"]) - async def test_key_router_settings_fallback_loads_per_user_tokens(self, monkeypatch, fallback_key): + async def test_key_router_settings_fallback_loads_per_user_tokens( + self, monkeypatch: pytest.MonkeyPatch, fallback_key: str + ): """The proxy forwards key/team router_settings fallback lists into the request after discovery, so discovery must read every one of them.""" from litellm.proxy.litellm_pre_call_utils import _resolve_user_provider_credentials_for_request diff --git a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py index 6860b477621..9332e5af086 100644 --- a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py +++ b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py @@ -5656,16 +5656,56 @@ async def test_db_content_policy_fallbacks_layer_over_the_litellm_settings_rules @pytest.mark.asyncio -@pytest.mark.parametrize("stored_rules", ["oops", [{"primary": "backup"}], [["backup"]]]) -async def test_invalid_db_content_policy_fallbacks_keep_the_router_rules(stored_rules: object): - router: Final = _content_policy_router(content_policy_fallbacks=[{"primary": ["backup"]}]) +@pytest.mark.parametrize( + "invalid_rule", + [{"secondary": "backup"}, ["backup"], dict[str, list[str]](), {"secondary": ["backup"], "primary": ["secondary"]}], +) +async def test_an_invalid_db_content_policy_rule_skips_only_itself(monkeypatch: pytest.MonkeyPatch, invalid_rule: object): + monkeypatch.setattr(litellm, "content_policy_fallbacks", [{"primary": ["backup"]}]) + router: Final = _content_policy_router() await proxy_server_module.ProxyConfig()._add_router_settings_from_db_config( - llm_router=router, prisma_client=_router_settings_row(stored_rules) + llm_router=router, prisma_client=_router_settings_row([invalid_rule, {"secondary": ["backup"]}]) ) assert router.allowed_fails == 7 assert await _answer_after_content_policy_refusal(router) == "backup answered" + assert await _answer_after_content_policy_refusal(router, "secondary") == "backup answered" + + +@pytest.mark.asyncio +async def test_a_db_content_policy_value_that_is_not_a_list_keeps_the_litellm_settings_rules( + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setattr(litellm, "content_policy_fallbacks", [{"primary": ["backup"]}]) + router: Final = _content_policy_router() + + await proxy_server_module.ProxyConfig()._add_router_settings_from_db_config( + llm_router=router, prisma_client=_router_settings_row("oops") + ) + + assert router.allowed_fails == 7 + assert await _answer_after_content_policy_refusal(router) == "backup answered" + + +@pytest.mark.asyncio +async def test_config_file_content_policy_fallbacks_stay_the_only_rules_after_a_db_sync( + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setattr(litellm, "content_policy_fallbacks", [{"secondary": ["backup"]}]) + file_rules: Final = [{"primary": ["backup"]}] + router: Final = _content_policy_router(content_policy_fallbacks=file_rules) + proxy_config: Final = proxy_server_module.ProxyConfig() + proxy_config.router_settings.load_yaml({"content_policy_fallbacks": file_rules}) + + await proxy_config._add_router_settings_from_db_config( + llm_router=router, prisma_client=_router_settings_row([{"secondary": ["backup"]}]) + ) + + assert router.allowed_fails == 7 + assert await _answer_after_content_policy_refusal(router) == "backup answered" + with pytest.raises(litellm.ContentPolicyViolationError): + await _answer_after_content_policy_refusal(router, "secondary") @pytest.mark.asyncio @@ -5714,7 +5754,7 @@ async def test_add_router_settings_from_db_config_empty_db_lists_do_not_clobber_ combined_settings = ChainMap(*(call.kwargs for call in reversed(mock_router.update_settings.call_args_list))) assert combined_settings["fallbacks"] == [{"gpt-oss-120b": ["granite-4-h-small"]}] assert combined_settings["context_window_fallbacks"] == [{"gpt-oss-120b": ["granite-4-h-small"]}] - assert combined_settings["content_policy_fallbacks"] == [{"gpt-oss-120b": ["granite-4-h-small"]}] + assert combined_settings.get("content_policy_fallbacks") != [{"gpt-oss-120b": ["other-model"]}] assert combined_settings["num_retries"] == 3 @@ -10055,7 +10095,15 @@ def test_update_config_validates_content_policy_fallbacks_like_fallbacks( ): client, prisma, restore = _update_config_setup(initial_rows={"router_settings": {"num_retries": 2}}) rule: Final = [{"gpt-5.4-mini": ["gpt-5.4-nano", {"model": "gpt-5.4-nano", "temperature": 1}]}] - malformed: Final = ("oops", 5, [1, 2], [{"gpt-5.4-mini": "gpt-5.4-nano"}], [{"gpt-5.4-mini": ["gpt-5.4-nano", 7]}]) + malformed: Final = ( + "oops", + 5, + [1, 2], + [{"gpt-5.4-mini": "gpt-5.4-nano"}], + [{"gpt-5.4-mini": ["gpt-5.4-nano", 7]}], + [dict[str, list[str]]()], + [{"gpt-5.4-mini": ["gpt-5.4-nano"], "gpt-5.4": ["gpt-5.4-nano"]}], + ) try: rejected: Final = tuple( client.post("/config/update", json={"router_settings": {"content_policy_fallbacks": value}}).status_code From 267dfe3d3f8423e74ae99d9720e40977963e0bdf Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 07:54:18 -0700 Subject: [PATCH 7/9] fix(proxy): answer an error for a string-valued fallback rule on GET /fallback instead of its characters --- .../management_endpoints/fallback_management_endpoints.py | 7 +++++-- litellm/types/router.py | 5 ++--- tests/unit/proxy/test_fallback_management_endpoints.py | 8 ++++++++ 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/management_endpoints/fallback_management_endpoints.py b/litellm/proxy/management_endpoints/fallback_management_endpoints.py index 6850e0d5bdc..d082174227d 100644 --- a/litellm/proxy/management_endpoints/fallback_management_endpoints.py +++ b/litellm/proxy/management_endpoints/fallback_management_endpoints.py @@ -43,7 +43,7 @@ from litellm.types.management_endpoints.router_settings_endpoints import ( FallbackGetResponse, FallbackResponse, ) -from litellm.types.router import ContentPolicyFallbackRule, ContentPolicyFallbackRules +from litellm.types.router import ContentPolicyFallbackRule, ContentPolicyFallbackRules, FallbackTargets router: Final = APIRouter() @@ -54,6 +54,7 @@ _STORED_FALLBACKS: Final = TypeAdapter(list[StoredFallback]) _JSON_RULES: Final = TypeAdapter(list[JsonValue]) _OPTIONAL_JSON_RULES: Final[TypeAdapter[Sequence[JsonValue] | None]] = TypeAdapter(Sequence[JsonValue] | None) _CONTENT_POLICY_FALLBACK_RULE: Final[TypeAdapter[ContentPolicyFallbackRule]] = TypeAdapter(ContentPolicyFallbackRule) +_FALLBACK_TARGETS: Final[TypeAdapter[FallbackTargets]] = TypeAdapter(FallbackTargets) def _rule_covers(entry: StoredFallback, model: str) -> bool: @@ -307,7 +308,9 @@ async def get_fallback( ) # Get fallbacks using the existing utility function - fallback_targets: Final = get_all_fallbacks(model=model, llm_router=llm_router, fallback_type=fallback_type) + fallback_targets: Final = _FALLBACK_TARGETS.validate_python( + get_all_fallbacks(model=model, llm_router=llm_router, fallback_type=fallback_type) + ) fallback_models: Final = [ name for target in fallback_targets if (name := fallback_target_model_name(target)) is not None ] diff --git a/litellm/types/router.py b/litellm/types/router.py index c11508bc350..ef8baf458e7 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -190,9 +190,8 @@ OptionalPreCallChecks = list[ ] -ContentPolicyFallbackRule: TypeAlias = Annotated[ - Mapping[str, Sequence[str | Mapping[str, JsonValue]]], Field(min_length=1, max_length=1) -] +FallbackTargets: TypeAlias = Sequence[str | Mapping[str, JsonValue]] +ContentPolicyFallbackRule: TypeAlias = Annotated[Mapping[str, FallbackTargets], Field(min_length=1, max_length=1)] ContentPolicyFallbackRules: TypeAlias = Sequence[ContentPolicyFallbackRule] diff --git a/tests/unit/proxy/test_fallback_management_endpoints.py b/tests/unit/proxy/test_fallback_management_endpoints.py index 2dc376bb4f3..8ebe762c36c 100644 --- a/tests/unit/proxy/test_fallback_management_endpoints.py +++ b/tests/unit/proxy/test_fallback_management_endpoints.py @@ -392,6 +392,14 @@ class TestFallbackWritesSeeTheLatestStoredRules: assert response.fallback_models == ["gpt-5.4-mini", "team-fallback"] + async def test_get_answers_an_error_for_a_string_valued_rule_instead_of_its_characters(self) -> None: + router: Final = _team_router() + router.update_settings(fallbacks=[{"team-primary": "team-fallback"}]) + with patch("litellm.proxy.proxy_server.llm_router", router), pytest.raises(HTTPException) as raised: + await get_fallback("team-primary", "general", MagicMock()) + + assert raised.value.status_code == 500 + async def test_a_create_keeps_a_rule_another_instance_stored_since_this_one_last_read(self) -> None: stored: Final = _StoredRouterSettings({}) await get_config_param(stored.prisma_client, "router_settings") From e43ea0dc22d7cc145541fbe688971eb98171465f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 12:07:09 -0700 Subject: [PATCH 8/9] fix(proxy): keep config-file content-policy rules as Router init accepted them and log a skipped rule once per change --- .../fallback_management_endpoints.py | 47 ++++++++++++------- ...test_proxy_server_endpoints_and_startup.py | 43 +++++++++++++++++ 2 files changed, 74 insertions(+), 16 deletions(-) diff --git a/litellm/proxy/management_endpoints/fallback_management_endpoints.py b/litellm/proxy/management_endpoints/fallback_management_endpoints.py index 465fc912e26..f6e7bd2a906 100644 --- a/litellm/proxy/management_endpoints/fallback_management_endpoints.py +++ b/litellm/proxy/management_endpoints/fallback_management_endpoints.py @@ -63,45 +63,60 @@ def _rule_covers(entry: StoredFallback, model: str) -> bool: def _layered_over_config_rules( - runtime_rules: ContentPolicyFallbackRules, config_rules: ContentPolicyFallbackRules -) -> ContentPolicyFallbackRules: + runtime_rules: ContentPolicyFallbackRules, config_rules: Sequence[Mapping[str, JsonValue]] +) -> Sequence[Mapping[str, object]]: covered: Final = frozenset[str]().union(*runtime_rules) return [*runtime_rules, *(rule for rule in config_rules if covered.isdisjoint(rule))] +def _config_file_rules(rules: object) -> Sequence[Mapping[str, JsonValue]]: + try: + candidates: Final = _OPTIONAL_JSON_RULES.validate_python(rules) or () + except ValidationError: + return () + return tuple(rule for rule in candidates if isinstance(rule, Mapping)) + + def _log_ignored_content_policy_rules(rules: object, error: ValidationError) -> None: verbose_proxy_logger.error( - "Ignoring invalid content_policy_fallbacks %r from config/DB, every other rule still applies. " - "Fix it through POST /fallback, /config/update, or the config file: %s", + "Ignoring invalid content_policy_fallbacks %r from the DB, every other rule still applies. " + "Fix it through POST /fallback or /config/update: %s", rules, error, ) -def _valid_content_policy_rule(rule: JsonValue) -> ContentPolicyFallbackRule | None: +def _checked_content_policy_rule(rule: JsonValue) -> ContentPolicyFallbackRule | ValidationError: try: return _CONTENT_POLICY_FALLBACK_RULE.validate_python(rule) except ValidationError as invalid_rule: - _log_ignored_content_policy_rules(rule, invalid_rule) - return None + return invalid_rule -def _valid_content_policy_rules(rules: object) -> ContentPolicyFallbackRules: +def _checked_content_policy_rules( + rules: object, +) -> Sequence[tuple[object, ContentPolicyFallbackRule | ValidationError]]: try: candidates: Final = _OPTIONAL_JSON_RULES.validate_python(rules) or () except ValidationError as not_a_rule_list: - _log_ignored_content_policy_rules(rules, not_a_rule_list) - return () - validated: Final = (_valid_content_policy_rule(candidate) for candidate in candidates) - return tuple(rule for rule in validated if rule is not None) + return ((rules, not_a_rule_list),) + return tuple((candidate, _checked_content_policy_rule(candidate)) for candidate in candidates) def apply_content_policy_fallbacks(llm_router: "Router", rules: object) -> None: - runtime_rules: Final = _valid_content_policy_rules(rules) - config_rules: Final = _valid_content_policy_rules( - litellm.content_policy_fallbacks # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # untyped module list + checked_rules: Final = _checked_content_policy_rules(rules) + runtime_rules: Final = tuple(result for _, result in checked_rules if not isinstance(result, ValidationError)) + layered_rules: Final = _layered_over_config_rules( + runtime_rules, + _config_file_rules( + litellm.content_policy_fallbacks # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # untyped module list + ), ) - layered_rules: Final = _layered_over_config_rules(runtime_rules, config_rules) + if layered_rules == llm_router.content_policy_fallbacks: # pyright: ignore[reportUnknownMemberType] # untyped Router list + return + for rule, result in checked_rules: + if isinstance(result, ValidationError): + _log_ignored_content_policy_rules(rule, result) llm_router.update_settings(content_policy_fallbacks=layered_rules) # pyright: ignore[reportUnknownMemberType] # untyped kwargs diff --git a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py index d032811e264..e6b24b68577 100644 --- a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py +++ b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py @@ -5688,6 +5688,49 @@ async def test_a_db_content_policy_value_that_is_not_a_list_keeps_the_litellm_se assert await _answer_after_content_policy_refusal(router) == "backup answered" +@pytest.mark.asyncio +async def test_a_db_sync_keeps_a_litellm_settings_rule_with_a_target_router_init_accepted( + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setattr(litellm, "content_policy_fallbacks", [{"primary": ["backup", 5]}]) + router: Final = _content_policy_router() + + await proxy_server_module.ProxyConfig()._add_router_settings_from_db_config( + llm_router=router, prisma_client=_router_settings_row([{"secondary": ["backup"]}]) + ) + + assert await _answer_after_content_policy_refusal(router) == "backup answered" + assert await _answer_after_content_policy_refusal(router, "secondary") == "backup answered" + + +@pytest.mark.asyncio +async def test_an_invalid_db_content_policy_rule_is_logged_once_per_change_of_the_rules( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +): + monkeypatch.setattr(litellm, "content_policy_fallbacks", []) + router: Final = _content_policy_router() + proxy_config: Final = proxy_server_module.ProxyConfig() + stored_rules: Final = [{"secondary": "backup"}, {"primary": ["backup"]}] + changed_rules: Final = [{"secondary": "backup"}, {"secondary": ["backup"]}] + + with caplog.at_level(logging.ERROR, logger="LiteLLM Proxy"): + await proxy_config._add_router_settings_from_db_config( + llm_router=router, prisma_client=_router_settings_row(stored_rules) + ) + await proxy_config._add_router_settings_from_db_config( + llm_router=router, prisma_client=_router_settings_row(stored_rules) + ) + await proxy_config._add_router_settings_from_db_config( + llm_router=router, prisma_client=_router_settings_row(changed_rules) + ) + + notices: Final = [ + record for record in caplog.records if "Ignoring invalid content_policy_fallbacks" in record.getMessage() + ] + assert len(notices) == 2 + assert await _answer_after_content_policy_refusal(router, "secondary") == "backup answered" + + @pytest.mark.asyncio async def test_config_file_content_policy_fallbacks_stay_the_only_rules_after_a_db_sync( monkeypatch: pytest.MonkeyPatch, From 835b953587f71bf8bf827976f057fd4fed731ae3 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 19:05:54 -0700 Subject: [PATCH 9/9] fix(proxy): log a skipped content-policy rule whenever the saved rules change --- .../fallback_management_endpoints.py | 10 ++++---- litellm/proxy/proxy_server.py | 14 +++++++++-- ...test_proxy_server_endpoints_and_startup.py | 25 +++++++++++++++++++ 3 files changed, 42 insertions(+), 7 deletions(-) diff --git a/litellm/proxy/management_endpoints/fallback_management_endpoints.py b/litellm/proxy/management_endpoints/fallback_management_endpoints.py index 821354cda31..b39afeea86b 100644 --- a/litellm/proxy/management_endpoints/fallback_management_endpoints.py +++ b/litellm/proxy/management_endpoints/fallback_management_endpoints.py @@ -104,7 +104,7 @@ def _checked_content_policy_rules( return tuple((candidate, _checked_content_policy_rule(candidate)) for candidate in candidates) -def apply_content_policy_fallbacks(llm_router: "Router", rules: object) -> None: +def apply_content_policy_fallbacks(llm_router: "Router", rules: object, *, log_skipped_rules: bool) -> None: checked_rules: Final = _checked_content_policy_rules(rules) runtime_rules: Final = tuple(result for _, result in checked_rules if not isinstance(result, ValidationError)) layered_rules: Final = _layered_over_config_rules( @@ -113,17 +113,17 @@ def apply_content_policy_fallbacks(llm_router: "Router", rules: object) -> None: litellm.content_policy_fallbacks # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # untyped module list ), ) + for rule, result in checked_rules: + if log_skipped_rules and isinstance(result, ValidationError): + _log_ignored_content_policy_rules(rule, result) if layered_rules == llm_router.settings().content_policy_fallbacks: # pyright: ignore[reportUnknownMemberType] # untyped RouterSettings list return - for rule, result in checked_rules: - if isinstance(result, ValidationError): - _log_ignored_content_policy_rules(rule, result) llm_router.apply(ChangeSet(settings={"content_policy_fallbacks": layered_rules})) def _apply_fallback_rules(llm_router: "Router", fallback_key: str, rules: Sequence[StoredFallback]) -> None: if fallback_key == "content_policy_fallbacks": - apply_content_policy_fallbacks(llm_router, rules) + apply_content_policy_fallbacks(llm_router, rules, log_skipped_rules=True) return llm_router.apply(ChangeSet(assign={fallback_key: rules})) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 3a34ca11d09..dc133e66a4d 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -8159,6 +8159,7 @@ class ProxyConfig: if db_router_settings is not None and db_router_settings.param_value is not None else _EMPTY_SETTINGS_MAPPING ) + previous_content_policy_rules: Final = self.router_settings.resolved().get("content_policy_fallbacks") self.router_settings.apply_db_row("router_settings", db_values) combined_router_settings: Final = self.router_settings.resolved() if combined_router_settings: @@ -8166,11 +8167,18 @@ class ProxyConfig: llm_router, combined_router_settings, content_policy_owned_by_config=self.router_settings.owned_by_config("content_policy_fallbacks"), + content_policy_rules_changed=( + combined_router_settings.get("content_policy_fallbacks") != previous_content_policy_rules + ), ) @staticmethod def _apply_router_settings( - llm_router: Router, router_settings: Mapping[str, object], *, content_policy_owned_by_config: bool + llm_router: Router, + router_settings: Mapping[str, object], + *, + content_policy_owned_by_config: bool, + content_policy_rules_changed: bool, ) -> None: llm_router.apply( ChangeSet( @@ -8180,7 +8188,9 @@ class ProxyConfig: ) ) if "content_policy_fallbacks" in router_settings and not content_policy_owned_by_config: - apply_content_policy_fallbacks(llm_router, router_settings["content_policy_fallbacks"]) + apply_content_policy_fallbacks( + llm_router, router_settings["content_policy_fallbacks"], log_skipped_rules=content_policy_rules_changed + ) if "routing_groups" not in router_settings: return try: diff --git a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py index c77db89551d..1f679a94f37 100644 --- a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py +++ b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py @@ -5733,6 +5733,31 @@ async def test_an_invalid_db_content_policy_rule_is_logged_once_per_change_of_th assert await _answer_after_content_policy_refusal(router, "secondary") == "backup answered" +@pytest.mark.asyncio +async def test_a_db_sync_logs_a_new_invalid_content_policy_rule_when_no_valid_rule_changes( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +): + monkeypatch.setattr(litellm, "content_policy_fallbacks", []) + router: Final = _content_policy_router() + proxy_config: Final = proxy_server_module.ProxyConfig() + + with caplog.at_level(logging.ERROR, logger="LiteLLM Proxy"): + await proxy_config._add_router_settings_from_db_config( + llm_router=router, prisma_client=_router_settings_row([{"secondary": "backup"}]) + ) + await proxy_config._add_router_settings_from_db_config( + llm_router=router, prisma_client=_router_settings_row([{"secondary": "backup"}, {"primary": "backup"}]) + ) + + notices: Final = [ + record.getMessage() + for record in caplog.records + if "Ignoring invalid content_policy_fallbacks" in record.getMessage() + ] + assert len(notices) == 3 + assert "{'primary': 'backup'}" in notices[-1] + + @pytest.mark.asyncio async def test_config_file_content_policy_fallbacks_stay_the_only_rules_after_a_db_sync( monkeypatch: pytest.MonkeyPatch,