mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(proxy): keep litellm_settings content policy fallbacks, accept dict targets, skip invalid stored rules
Some checks failed
LiteLLM Rust DB / db-queries (push) Waiting to run
LiteLLM Rust / rust-lint (push) Waiting to run
LiteLLM Rust / rust-test (push) Waiting to run
LiteLLM Rust / rust-wheel (push) Waiting to run
Terraform Provider / gofmt, vet, build, test (push) Waiting to run
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Waiting to run
Terraform Modules / fmt, validate, test (aws) (push) Has been cancelled
Terraform Modules / fmt, validate, test (gcp) (push) Has been cancelled
Some checks failed
LiteLLM Rust DB / db-queries (push) Waiting to run
LiteLLM Rust / rust-lint (push) Waiting to run
LiteLLM Rust / rust-test (push) Waiting to run
LiteLLM Rust / rust-wheel (push) Waiting to run
Terraform Provider / gofmt, vet, build, test (push) Waiting to run
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Waiting to run
Terraform Modules / fmt, validate, test (aws) (push) Has been cancelled
Terraform Modules / fmt, validate, test (gcp) (push) Has been cancelled
A runtime content_policy_fallbacks rule now layers over the litellm_settings rules instead of replacing them, so models the runtime rules do not name keep their file fallbacks. Fallback targets may be deployment dicts as well as model names. A stored rule that fails validation is logged and skipped at sync, so the other router settings still apply. Per-user token discovery now reads every fallback key from key and team router_settings.
This commit is contained in:
parent
941a965bfc
commit
0a1fba1f5a
9 changed files with 187 additions and 21 deletions
|
|
@ -3139,9 +3139,9 @@ def _fallback_lists(
|
|||
) -> tuple[Sequence[object], ...]:
|
||||
"""Every fallback list the router can route this request through: the three
|
||||
router-level lists, the same three keys from the request body (which replace
|
||||
the router's when present, so the union is the safe superset), and the
|
||||
key/team ``router_settings.fallbacks`` the proxy prefers over router-level
|
||||
fallbacks in ``_configured_fallbacks``."""
|
||||
the router's when present, so the union is the safe superset), and the same
|
||||
three keys from key/team ``router_settings``, which the proxy forwards into
|
||||
the request after this discovery runs."""
|
||||
lists: Final[list[Sequence[object]]] = [] # mutable-ok: accumulates each fallback list found
|
||||
for router_list in (
|
||||
cast("object", llm_router.fallbacks), # cast-ok: router fallback attrs are untyped
|
||||
|
|
@ -3155,11 +3155,12 @@ def _fallback_lists(
|
|||
if isinstance(entries, list):
|
||||
lists.append(cast("Sequence[object]", entries)) # cast-ok: request fallback lists hold mixed entry shapes
|
||||
for settings in (router_settings, team_router_settings):
|
||||
key_fallbacks: object = settings.get("fallbacks") if settings is not None else None
|
||||
if isinstance(key_fallbacks, list):
|
||||
lists.append(
|
||||
cast("Sequence[object]", key_fallbacks) # cast-ok: settings fallback lists hold mixed entry shapes
|
||||
)
|
||||
for key in _REQUEST_FALLBACK_KEYS:
|
||||
key_fallbacks: object = settings.get(key) if settings is not None else None
|
||||
if isinstance(key_fallbacks, list):
|
||||
lists.append(
|
||||
cast("Sequence[object]", key_fallbacks) # cast-ok: settings fallback lists hold mixed entry shapes
|
||||
)
|
||||
return tuple(lists)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -14,8 +14,9 @@ import json
|
|||
from collections.abc import Mapping, Sequence
|
||||
from typing import TYPE_CHECKING, Annotated, Final, Literal
|
||||
|
||||
from pydantic import Field, JsonValue, TypeAdapter
|
||||
from pydantic import Field, JsonValue, TypeAdapter, ValidationError
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.auth.model_checks import get_all_fallbacks
|
||||
|
|
@ -26,6 +27,7 @@ if TYPE_CHECKING:
|
|||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
|
||||
from litellm.proxy.proxy_server import ProxyConfig
|
||||
from litellm.router import Router
|
||||
else:
|
||||
try:
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
|
|
@ -40,6 +42,7 @@ from litellm.types.management_endpoints.router_settings_endpoints import (
|
|||
FallbackGetResponse,
|
||||
FallbackResponse,
|
||||
)
|
||||
from litellm.types.router import ContentPolicyFallbackRules
|
||||
|
||||
router: Final = APIRouter()
|
||||
|
||||
|
|
@ -48,12 +51,44 @@ FallbackRule = dict[str, list[str]]
|
|||
StoredFallback = Annotated[FallbackRule | dict[str, object] | str, Field(union_mode="left_to_right")]
|
||||
_STORED_FALLBACKS: Final = TypeAdapter(list[StoredFallback])
|
||||
_JSON_RULES: Final = TypeAdapter(list[JsonValue])
|
||||
_CONTENT_POLICY_FALLBACK_RULES: Final = TypeAdapter(ContentPolicyFallbackRules)
|
||||
|
||||
|
||||
def _rule_covers(entry: StoredFallback, model: str) -> bool:
|
||||
return isinstance(entry, dict) and model in entry
|
||||
|
||||
|
||||
def _layered_over_config_rules(
|
||||
runtime_rules: ContentPolicyFallbackRules, config_rules: ContentPolicyFallbackRules
|
||||
) -> ContentPolicyFallbackRules:
|
||||
covered: Final = frozenset[str]().union(*runtime_rules)
|
||||
return [*runtime_rules, *(rule for rule in config_rules if covered.isdisjoint(rule))]
|
||||
|
||||
|
||||
def apply_content_policy_fallbacks(llm_router: "Router", rules: object) -> None:
|
||||
try:
|
||||
runtime_rules: Final = _CONTENT_POLICY_FALLBACK_RULES.validate_python(rules)
|
||||
config_rules: Final = _CONTENT_POLICY_FALLBACK_RULES.validate_python(
|
||||
litellm.content_policy_fallbacks or [] # pyright: ignore[reportUnknownMemberType] # untyped module list
|
||||
)
|
||||
except ValidationError as invalid_rules:
|
||||
verbose_proxy_logger.error(
|
||||
"Ignoring invalid content_policy_fallbacks from config/DB, all other router settings still apply. "
|
||||
"Fix the content policy fallbacks in the Admin UI or config to load them: %s",
|
||||
invalid_rules,
|
||||
)
|
||||
return
|
||||
layered_rules: Final = _layered_over_config_rules(runtime_rules, config_rules)
|
||||
llm_router.update_settings(content_policy_fallbacks=layered_rules) # pyright: ignore[reportUnknownMemberType] # untyped kwargs
|
||||
|
||||
|
||||
def _apply_fallback_rules(llm_router: "Router", fallback_key: str, rules: Sequence[StoredFallback]) -> None:
|
||||
if fallback_key == "content_policy_fallbacks":
|
||||
apply_content_policy_fallbacks(llm_router, rules)
|
||||
return
|
||||
setattr(llm_router, fallback_key, rules)
|
||||
|
||||
|
||||
async def _router_settings_fresh_from_db(proxy_config: "ProxyConfig") -> dict[str, object]:
|
||||
await evict_config_param(ROUTER_SETTINGS_PARAM)
|
||||
config: Final = await proxy_config.get_config()
|
||||
|
|
@ -195,7 +230,7 @@ async def create_fallback(
|
|||
await _persist_router_settings(prisma_client, router_settings)
|
||||
|
||||
# Update the in-memory router configuration
|
||||
setattr(llm_router, fallback_key, existing_fallbacks)
|
||||
_apply_fallback_rules(llm_router, fallback_key, existing_fallbacks)
|
||||
|
||||
verbose_proxy_logger.info(
|
||||
"Fallback configured: %s -> %s (type: %s)", data.model, data.fallback_models, data.fallback_type
|
||||
|
|
@ -348,7 +383,7 @@ async def delete_fallback(
|
|||
await _persist_router_settings(prisma_client, router_settings)
|
||||
|
||||
# Update the in-memory router configuration
|
||||
setattr(llm_router, fallback_key, updated_fallbacks)
|
||||
_apply_fallback_rules(llm_router, fallback_key, updated_fallbacks)
|
||||
|
||||
verbose_proxy_logger.info("Fallback deleted: %s (type: %s)", model, fallback_type)
|
||||
|
||||
|
|
|
|||
|
|
@ -702,6 +702,9 @@ from litellm.proxy.management_endpoints.customer_endpoints import (
|
|||
router as customer_router,
|
||||
)
|
||||
from litellm.proxy.management_endpoints.daily_activity_routes import router as daily_activity_router
|
||||
from litellm.proxy.management_endpoints.fallback_management_endpoints import (
|
||||
apply_content_policy_fallbacks,
|
||||
)
|
||||
from litellm.proxy.management_endpoints.fallback_management_endpoints import (
|
||||
router as fallback_management_router,
|
||||
)
|
||||
|
|
@ -8094,7 +8097,11 @@ class ProxyConfig:
|
|||
|
||||
@staticmethod
|
||||
def _apply_router_settings(llm_router: Router, router_settings: Mapping[str, object]) -> None:
|
||||
llm_router.update_settings(**{k: v for k, v in router_settings.items() if k != "routing_groups"})
|
||||
llm_router.update_settings(
|
||||
**{k: v for k, v in router_settings.items() if k not in ("routing_groups", "content_policy_fallbacks")}
|
||||
)
|
||||
if "content_policy_fallbacks" in router_settings:
|
||||
apply_content_policy_fallbacks(llm_router, router_settings["content_policy_fallbacks"])
|
||||
if "routing_groups" not in router_settings:
|
||||
return
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -185,6 +185,9 @@ OptionalPreCallChecks = list[
|
|||
]
|
||||
|
||||
|
||||
ContentPolicyFallbackRules: TypeAlias = Sequence[Mapping[str, Sequence[str | Mapping[str, JsonValue]]]]
|
||||
|
||||
|
||||
class UpdateRouterConfig(LiteLLMBaseModel):
|
||||
"""
|
||||
Set of params that you can modify via `router.update_settings()`.
|
||||
|
|
@ -204,7 +207,7 @@ class UpdateRouterConfig(LiteLLMBaseModel):
|
|||
retry_after: float | None = None
|
||||
fallbacks: list[dict] | None = None
|
||||
context_window_fallbacks: list[dict] | None = None
|
||||
content_policy_fallbacks: Sequence[Mapping[str, Sequence[str]]] | None = None
|
||||
content_policy_fallbacks: ContentPolicyFallbackRules | None = None
|
||||
model_group_alias: dict[str, str | dict] | None = Field(default={})
|
||||
enable_tag_filtering: bool | None = None
|
||||
weights: RouterWeights | None = None
|
||||
|
|
|
|||
|
|
@ -7597,6 +7597,7 @@ async def test_generate_key_with_router_settings(monkeypatch):
|
|||
"num_retries": 3,
|
||||
"model_group_retry_policy": {"gpt-4": {"RateLimitErrorRetries": 5}},
|
||||
"weights": {"gpt-4": {"weighted-id": 1}},
|
||||
"content_policy_fallbacks": [{"gpt-4": ["gpt-4o", {"model": "gpt-4o-mini", "temperature": 0}]}],
|
||||
}
|
||||
|
||||
request_data = GenerateKeyRequest(
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ import pytest
|
|||
from fastapi import HTTPException
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm import Router
|
||||
from litellm.proxy.management_endpoints.fallback_management_endpoints import (
|
||||
FallbackCreateRequest,
|
||||
|
|
@ -331,6 +332,31 @@ class TestFallbackWritesSeeTheLatestStoredRules:
|
|||
assert stored.stored_fallbacks() == [TEAM_RULE, GATEWAY_RULE]
|
||||
assert await stored.cached_fallbacks() == [TEAM_RULE, GATEWAY_RULE]
|
||||
|
||||
async def test_content_policy_writes_keep_the_litellm_settings_rules_for_other_models(
|
||||
self, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "content_policy_fallbacks", [TEAM_RULE])
|
||||
router: Final = _team_router()
|
||||
stored: Final = _StoredRouterSettings(None)
|
||||
with (
|
||||
patch("litellm.proxy.proxy_server.llm_router", router),
|
||||
patch("litellm.proxy.proxy_server.prisma_client", stored.prisma_client),
|
||||
patch("litellm.proxy.proxy_server.proxy_config", stored.proxy_config),
|
||||
patch("litellm.proxy.proxy_server.store_model_in_db", True),
|
||||
):
|
||||
await create_fallback(
|
||||
FallbackCreateRequest(
|
||||
model="gpt-5.4-mini", fallback_models=["team-fallback"], fallback_type="content_policy"
|
||||
),
|
||||
MagicMock(),
|
||||
)
|
||||
after_create: Final = await get_fallback("team-primary", "content_policy", MagicMock())
|
||||
await delete_fallback("gpt-5.4-mini", "content_policy", MagicMock())
|
||||
after_delete: Final = await get_fallback("team-primary", "content_policy", MagicMock())
|
||||
|
||||
assert after_create.fallback_models == ["team-fallback"]
|
||||
assert after_delete.fallback_models == ["team-fallback"]
|
||||
|
||||
async def test_a_create_keeps_a_rule_another_instance_stored_since_this_one_last_read(self) -> None:
|
||||
stored: Final = _StoredRouterSettings({})
|
||||
await get_config_param(stored.prisma_client, "router_settings")
|
||||
|
|
|
|||
|
|
@ -10071,9 +10071,10 @@ class TestResolveUserProviderCredentials:
|
|||
assert data["secret_fields"]["user_provider_credentials_user_id"] == "user-a"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_key_router_settings_fallback_loads_per_user_tokens(self, monkeypatch):
|
||||
"""The proxy prefers key/team router_settings.fallbacks over router-level
|
||||
fallbacks (_configured_fallbacks), so discovery must read them too."""
|
||||
@pytest.mark.parametrize("fallback_key", ["fallbacks", "context_window_fallbacks", "content_policy_fallbacks"])
|
||||
async def test_key_router_settings_fallback_loads_per_user_tokens(self, monkeypatch, fallback_key):
|
||||
"""The proxy forwards key/team router_settings fallback lists into the
|
||||
request after discovery, so discovery must read every one of them."""
|
||||
from litellm.proxy.litellm_pre_call_utils import _resolve_user_provider_credentials_for_request
|
||||
|
||||
table = self._env(monkeypatch)
|
||||
|
|
@ -10103,7 +10104,7 @@ class TestResolveUserProviderCredentials:
|
|||
authenticated_user_id="user-a",
|
||||
team_id=None,
|
||||
llm_router=router,
|
||||
router_settings={"fallbacks": [{"gpt-4o": [{"model": "copilot-chat"}]}]},
|
||||
router_settings={fallback_key: [{"gpt-4o": [{"model": "copilot-chat"}]}]},
|
||||
)
|
||||
table.find_many.assert_awaited()
|
||||
assert data["secret_fields"]["user_provider_credentials_user_id"] == "user-a"
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import subprocess
|
|||
import sys
|
||||
import time
|
||||
import types
|
||||
from collections.abc import Callable
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
|
@ -5581,6 +5582,92 @@ async def test_valid_db_routing_groups_still_replace_router_groups():
|
|||
assert router._get_routing_context("m2", None)[0] == "least-busy"
|
||||
|
||||
|
||||
def _content_policy_router(**router_kwargs: object) -> litellm.Router:
|
||||
return litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "primary",
|
||||
"litellm_params": {"model": "openai/gpt-5.4-mini", "api_key": "fake", "mock_response": "primary answered"},
|
||||
},
|
||||
{
|
||||
"model_name": "secondary",
|
||||
"litellm_params": {"model": "openai/gpt-5.4-mini", "api_key": "fake", "mock_response": "secondary answered"},
|
||||
},
|
||||
{
|
||||
"model_name": "backup",
|
||||
"litellm_params": {"model": "openai/gpt-5.4-nano", "api_key": "fake", "mock_response": "backup answered"},
|
||||
},
|
||||
],
|
||||
num_retries=0,
|
||||
**router_kwargs,
|
||||
)
|
||||
|
||||
|
||||
def _router_settings_row(content_policy_fallbacks: object) -> MagicMock:
|
||||
row: Final = MagicMock()
|
||||
row.param_value = {"allowed_fails": 7, "content_policy_fallbacks": content_policy_fallbacks}
|
||||
prisma: Final = MagicMock()
|
||||
prisma.db.litellm_config.find_first = AsyncMock(return_value=row)
|
||||
return prisma
|
||||
|
||||
|
||||
async def _answer_after_content_policy_refusal(router: litellm.Router, model: str = "primary") -> str | None:
|
||||
response: Final = await router.acompletion(
|
||||
model=model, messages=[{"role": "user", "content": "hi"}], mock_testing_content_policy_fallbacks=True
|
||||
)
|
||||
return response.choices[0].message.content
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("stored_target", ["backup", {"model": "backup"}])
|
||||
async def test_db_content_policy_fallbacks_reach_the_router(stored_target: str | dict[str, str]):
|
||||
router: Final = _content_policy_router()
|
||||
|
||||
await proxy_server_module.ProxyConfig()._add_router_settings_from_db_config(
|
||||
llm_router=router, prisma_client=_router_settings_row([{"primary": [stored_target]}])
|
||||
)
|
||||
|
||||
assert await _answer_after_content_policy_refusal(router) == "backup answered"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("litellm_settings_rules", "stored_rules", "refused_model"),
|
||||
[
|
||||
([{"secondary": ["backup"]}], [{"primary": ["backup"]}], "secondary"),
|
||||
([{"*": ["secondary"]}], [{"*": ["backup"]}], "primary"),
|
||||
([{"primary": ["backup"]}], [], "primary"),
|
||||
],
|
||||
)
|
||||
async def test_db_content_policy_fallbacks_layer_over_the_litellm_settings_rules(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
litellm_settings_rules: list[dict[str, list[str]]],
|
||||
stored_rules: list[dict[str, list[str]]],
|
||||
refused_model: str,
|
||||
):
|
||||
monkeypatch.setattr(litellm, "content_policy_fallbacks", litellm_settings_rules)
|
||||
router: Final = _content_policy_router()
|
||||
|
||||
await proxy_server_module.ProxyConfig()._add_router_settings_from_db_config(
|
||||
llm_router=router, prisma_client=_router_settings_row(stored_rules)
|
||||
)
|
||||
|
||||
assert await _answer_after_content_policy_refusal(router, refused_model) == "backup answered"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("stored_rules", ["oops", [{"primary": "backup"}], [["backup"]]])
|
||||
async def test_invalid_db_content_policy_fallbacks_keep_the_router_rules(stored_rules: object):
|
||||
router: Final = _content_policy_router(content_policy_fallbacks=[{"primary": ["backup"]}])
|
||||
|
||||
await proxy_server_module.ProxyConfig()._add_router_settings_from_db_config(
|
||||
llm_router=router, prisma_client=_router_settings_row(stored_rules)
|
||||
)
|
||||
|
||||
assert router.allowed_fails == 7
|
||||
assert await _answer_after_content_policy_refusal(router) == "backup answered"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_router_settings_from_db_config_empty_db_lists_do_not_clobber_config_fallbacks():
|
||||
"""
|
||||
|
|
@ -5589,6 +5676,7 @@ async def test_add_router_settings_from_db_config_empty_db_lists_do_not_clobber_
|
|||
empty lists are "no value" and must not clobber config.yaml fallbacks,
|
||||
matching _deep_merge_dicts semantics. Non-empty DB lists still win.
|
||||
"""
|
||||
from collections import ChainMap
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
from litellm.proxy.proxy_server import ProxyConfig
|
||||
|
|
@ -5623,7 +5711,7 @@ async def test_add_router_settings_from_db_config_empty_db_lists_do_not_clobber_
|
|||
prisma_client=mock_prisma_client,
|
||||
)
|
||||
|
||||
combined_settings = mock_router.update_settings.call_args.kwargs
|
||||
combined_settings = ChainMap(*(call.kwargs for call in reversed(mock_router.update_settings.call_args_list)))
|
||||
assert combined_settings["fallbacks"] == [{"gpt-oss-120b": ["granite-4-h-small"]}]
|
||||
assert combined_settings["context_window_fallbacks"] == [{"gpt-oss-120b": ["granite-4-h-small"]}]
|
||||
assert combined_settings["content_policy_fallbacks"] == [{"gpt-oss-120b": ["granite-4-h-small"]}]
|
||||
|
|
@ -9962,9 +10050,11 @@ def test_update_config_accepts_disjoint_routing_groups(_update_config_setup):
|
|||
restore()
|
||||
|
||||
|
||||
def test_update_config_validates_content_policy_fallbacks_like_fallbacks(_update_config_setup):
|
||||
def test_update_config_validates_content_policy_fallbacks_like_fallbacks(
|
||||
_update_config_setup: Callable[..., tuple[TestClient, _FakePrismaClient, Callable[[], None]]],
|
||||
):
|
||||
client, prisma, restore = _update_config_setup(initial_rows={"router_settings": {"num_retries": 2}})
|
||||
rule: Final = [{"gpt-5.4-mini": ["gpt-5.4-nano"]}]
|
||||
rule: Final = [{"gpt-5.4-mini": ["gpt-5.4-nano", {"model": "gpt-5.4-nano", "temperature": 1}]}]
|
||||
malformed: Final = ("oops", 5, [1, 2], [{"gpt-5.4-mini": "gpt-5.4-nano"}], [{"gpt-5.4-mini": ["gpt-5.4-nano", 7]}])
|
||||
try:
|
||||
rejected: Final = tuple(
|
||||
|
|
|
|||
4
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
4
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -48688,7 +48688,9 @@ export interface components {
|
|||
allowed_fails?: number | null;
|
||||
/** Content Policy Fallbacks */
|
||||
content_policy_fallbacks?: {
|
||||
[key: string]: string[];
|
||||
[key: string]: (string | {
|
||||
[key: string]: components["schemas"]["JsonValue"];
|
||||
})[];
|
||||
}[] | null;
|
||||
/** Context Window Fallbacks */
|
||||
context_window_fallbacks?: {
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue