mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
Merge 835b953587 into 85af2f65c9
This commit is contained in:
commit
6dabc68b4c
11 changed files with 488 additions and 28 deletions
|
|
@ -67,6 +67,7 @@ RUNTIME_UPDATABLE_ROUTER_SETTINGS: Final[frozenset[str]] = frozenset(
|
|||
"retry_after",
|
||||
"fallbacks",
|
||||
"context_window_fallbacks",
|
||||
"content_policy_fallbacks",
|
||||
"retry_policy",
|
||||
"model_group_retry_policy",
|
||||
"model_group_alias",
|
||||
|
|
|
|||
|
|
@ -3138,9 +3138,9 @@ def _fallback_lists(
|
|||
) -> tuple[Sequence[object], ...]:
|
||||
"""Every fallback list the router can route this request through: the three
|
||||
router-level lists, the same three keys from the request body (which replace
|
||||
the router's when present, so the union is the safe superset), and the
|
||||
key/team ``router_settings.fallbacks`` the proxy prefers over router-level
|
||||
fallbacks in ``_configured_fallbacks``."""
|
||||
the router's when present, so the union is the safe superset), and the same
|
||||
three keys from key/team ``router_settings``, which the proxy forwards into
|
||||
the request after this discovery runs."""
|
||||
lists: Final[list[Sequence[object]]] = [] # mutable-ok: accumulates each fallback list found
|
||||
configured: Final = llm_router.settings()
|
||||
for router_list in (
|
||||
|
|
@ -3155,11 +3155,12 @@ def _fallback_lists(
|
|||
if isinstance(entries, list):
|
||||
lists.append(cast("Sequence[object]", entries)) # cast-ok: request fallback lists hold mixed entry shapes
|
||||
for settings in (router_settings, team_router_settings):
|
||||
key_fallbacks: object = settings.get("fallbacks") if settings is not None else None
|
||||
if isinstance(key_fallbacks, list):
|
||||
lists.append(
|
||||
cast("Sequence[object]", key_fallbacks) # cast-ok: settings fallback lists hold mixed entry shapes
|
||||
)
|
||||
for key in _REQUEST_FALLBACK_KEYS:
|
||||
key_fallbacks: object = settings.get(key) if settings is not None else None
|
||||
if isinstance(key_fallbacks, list):
|
||||
lists.append(
|
||||
cast("Sequence[object]", key_fallbacks) # cast-ok: settings fallback lists hold mixed entry shapes
|
||||
)
|
||||
return tuple(lists)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -14,10 +14,12 @@ import json
|
|||
from collections.abc import Mapping, Sequence
|
||||
from typing import TYPE_CHECKING, Annotated, Final, Literal
|
||||
|
||||
from pydantic import Field, JsonValue, TypeAdapter
|
||||
from pydantic import Field, JsonValue, TypeAdapter, ValidationError
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.auth.auth_utils import fallback_target_model_name
|
||||
from litellm.proxy.auth.model_checks import get_all_fallbacks
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.utils import PrismaClient, invalidate_config_param
|
||||
|
|
@ -26,6 +28,7 @@ if TYPE_CHECKING:
|
|||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
|
||||
from litellm.proxy.proxy_server import ProxyConfig
|
||||
from litellm.router import Router
|
||||
else:
|
||||
try:
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
|
|
@ -41,6 +44,7 @@ from litellm.types.management_endpoints.router_settings_endpoints import (
|
|||
FallbackGetResponse,
|
||||
FallbackResponse,
|
||||
)
|
||||
from litellm.types.router import ContentPolicyFallbackRule, ContentPolicyFallbackRules, FallbackTargets
|
||||
|
||||
router: Final = APIRouter()
|
||||
|
||||
|
|
@ -50,12 +54,80 @@ StoredFallback = Annotated[FallbackRule | dict[str, object] | str, Field(union_m
|
|||
_STORED_FALLBACKS: Final = TypeAdapter(list[StoredFallback])
|
||||
_JSON_RULES: Final = TypeAdapter(list[JsonValue])
|
||||
_STORED_ROUTER_SETTINGS: Final = TypeAdapter(dict[str, JsonValue])
|
||||
_OPTIONAL_JSON_RULES: Final[TypeAdapter[Sequence[JsonValue] | None]] = TypeAdapter(Sequence[JsonValue] | None)
|
||||
_CONTENT_POLICY_FALLBACK_RULE: Final[TypeAdapter[ContentPolicyFallbackRule]] = TypeAdapter(ContentPolicyFallbackRule)
|
||||
_FALLBACK_TARGETS: Final[TypeAdapter[FallbackTargets]] = TypeAdapter(FallbackTargets)
|
||||
|
||||
|
||||
def _rule_covers(entry: StoredFallback, model: str) -> bool:
|
||||
return isinstance(entry, dict) and model in entry
|
||||
|
||||
|
||||
def _layered_over_config_rules(
|
||||
runtime_rules: ContentPolicyFallbackRules, config_rules: Sequence[Mapping[str, JsonValue]]
|
||||
) -> Sequence[Mapping[str, object]]:
|
||||
covered: Final = frozenset[str]().union(*runtime_rules)
|
||||
return [*runtime_rules, *(rule for rule in config_rules if covered.isdisjoint(rule))]
|
||||
|
||||
|
||||
def _config_file_rules(rules: object) -> Sequence[Mapping[str, JsonValue]]:
|
||||
try:
|
||||
candidates: Final = _OPTIONAL_JSON_RULES.validate_python(rules) or ()
|
||||
except ValidationError:
|
||||
return ()
|
||||
return tuple(rule for rule in candidates if isinstance(rule, Mapping))
|
||||
|
||||
|
||||
def _log_ignored_content_policy_rules(rules: object, error: ValidationError) -> None:
|
||||
verbose_proxy_logger.error(
|
||||
"Ignoring invalid content_policy_fallbacks %r from the DB, every other rule still applies. "
|
||||
"Fix it through POST /fallback or /config/update: %s",
|
||||
rules,
|
||||
error,
|
||||
)
|
||||
|
||||
|
||||
def _checked_content_policy_rule(rule: JsonValue) -> ContentPolicyFallbackRule | ValidationError:
|
||||
try:
|
||||
return _CONTENT_POLICY_FALLBACK_RULE.validate_python(rule)
|
||||
except ValidationError as invalid_rule:
|
||||
return invalid_rule
|
||||
|
||||
|
||||
def _checked_content_policy_rules(
|
||||
rules: object,
|
||||
) -> Sequence[tuple[object, ContentPolicyFallbackRule | ValidationError]]:
|
||||
try:
|
||||
candidates: Final = _OPTIONAL_JSON_RULES.validate_python(rules) or ()
|
||||
except ValidationError as not_a_rule_list:
|
||||
return ((rules, not_a_rule_list),)
|
||||
return tuple((candidate, _checked_content_policy_rule(candidate)) for candidate in candidates)
|
||||
|
||||
|
||||
def apply_content_policy_fallbacks(llm_router: "Router", rules: object, *, log_skipped_rules: bool) -> None:
|
||||
checked_rules: Final = _checked_content_policy_rules(rules)
|
||||
runtime_rules: Final = tuple(result for _, result in checked_rules if not isinstance(result, ValidationError))
|
||||
layered_rules: Final = _layered_over_config_rules(
|
||||
runtime_rules,
|
||||
_config_file_rules(
|
||||
litellm.content_policy_fallbacks # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # untyped module list
|
||||
),
|
||||
)
|
||||
for rule, result in checked_rules:
|
||||
if log_skipped_rules and isinstance(result, ValidationError):
|
||||
_log_ignored_content_policy_rules(rule, result)
|
||||
if layered_rules == llm_router.settings().content_policy_fallbacks: # pyright: ignore[reportUnknownMemberType] # untyped RouterSettings list
|
||||
return
|
||||
llm_router.apply(ChangeSet(settings={"content_policy_fallbacks": layered_rules}))
|
||||
|
||||
|
||||
def _apply_fallback_rules(llm_router: "Router", fallback_key: str, rules: Sequence[StoredFallback]) -> None:
|
||||
if fallback_key == "content_policy_fallbacks":
|
||||
apply_content_policy_fallbacks(llm_router, rules, log_skipped_rules=True)
|
||||
return
|
||||
llm_router.apply(ChangeSet(assign={fallback_key: rules}))
|
||||
|
||||
|
||||
async def _reload_config_file_ownership(proxy_config: "ProxyConfig") -> None:
|
||||
await proxy_config.get_config()
|
||||
|
||||
|
|
@ -83,6 +155,7 @@ def _editable_rules(
|
|||
async def _write_rules(
|
||||
prisma_client: PrismaClient,
|
||||
proxy_config: "ProxyConfig",
|
||||
llm_router: "Router",
|
||||
stored_settings: Mapping[str, JsonValue],
|
||||
fallback_key: str,
|
||||
rules: Sequence[StoredFallback],
|
||||
|
|
@ -91,8 +164,10 @@ async def _write_rules(
|
|||
section_name=ROUTER_SETTINGS_PARAM, changed_keys={fallback_key: _JSON_RULES.validate_python(rules)}
|
||||
)
|
||||
if proxy_config.router_settings.owned_by_config(fallback_key):
|
||||
llm_router.apply(ChangeSet(assign={fallback_key: rules}))
|
||||
return
|
||||
await _persist_router_settings(prisma_client, {**stored_settings, fallback_key: rules})
|
||||
_apply_fallback_rules(llm_router, fallback_key, rules)
|
||||
|
||||
|
||||
async def _persist_router_settings(prisma_client: PrismaClient, router_settings: Mapping[str, object]) -> None:
|
||||
|
|
@ -218,10 +293,7 @@ async def create_fallback(
|
|||
# Add new fallback
|
||||
existing_fallbacks.append({data.model: data.fallback_models})
|
||||
|
||||
await _write_rules(prisma_client, proxy_config, router_settings, fallback_key, existing_fallbacks)
|
||||
|
||||
# Update the in-memory router configuration
|
||||
llm_router.apply(ChangeSet(assign={fallback_key: existing_fallbacks}))
|
||||
await _write_rules(prisma_client, proxy_config, llm_router, router_settings, fallback_key, existing_fallbacks)
|
||||
|
||||
verbose_proxy_logger.info(
|
||||
"Fallback configured: %s -> %s (type: %s)", data.model, data.fallback_models, data.fallback_type
|
||||
|
|
@ -277,7 +349,12 @@ async def get_fallback(
|
|||
)
|
||||
|
||||
# Get fallbacks using the existing utility function
|
||||
fallback_models: Final = get_all_fallbacks(model=model, llm_router=llm_router, fallback_type=fallback_type)
|
||||
fallback_targets: Final = _FALLBACK_TARGETS.validate_python(
|
||||
get_all_fallbacks(model=model, llm_router=llm_router, fallback_type=fallback_type)
|
||||
)
|
||||
fallback_models: Final = [
|
||||
name for target in fallback_targets if (name := fallback_target_model_name(target)) is not None
|
||||
]
|
||||
|
||||
if not fallback_models:
|
||||
raise HTTPException(
|
||||
|
|
@ -367,10 +444,7 @@ async def delete_fallback(
|
|||
detail={"error": f"No {fallback_type} fallbacks configured for model '{model}'"},
|
||||
)
|
||||
|
||||
await _write_rules(prisma_client, proxy_config, router_settings, fallback_key, updated_fallbacks)
|
||||
|
||||
# Update the in-memory router configuration
|
||||
llm_router.apply(ChangeSet(assign={fallback_key: updated_fallbacks}))
|
||||
await _write_rules(prisma_client, proxy_config, llm_router, router_settings, fallback_key, updated_fallbacks)
|
||||
|
||||
verbose_proxy_logger.info("Fallback deleted: %s (type: %s)", model, fallback_type)
|
||||
|
||||
|
|
|
|||
|
|
@ -717,6 +717,9 @@ from litellm.proxy.management_endpoints.customer_endpoints import (
|
|||
router as customer_router,
|
||||
)
|
||||
from litellm.proxy.management_endpoints.daily_activity_routes import router as daily_activity_router
|
||||
from litellm.proxy.management_endpoints.fallback_management_endpoints import (
|
||||
apply_content_policy_fallbacks,
|
||||
)
|
||||
from litellm.proxy.management_endpoints.fallback_management_endpoints import (
|
||||
router as fallback_management_router,
|
||||
)
|
||||
|
|
@ -8159,14 +8162,38 @@ class ProxyConfig:
|
|||
if db_router_settings is not None and db_router_settings.param_value is not None
|
||||
else _EMPTY_SETTINGS_MAPPING
|
||||
)
|
||||
previous_content_policy_rules: Final = self.router_settings.resolved().get("content_policy_fallbacks")
|
||||
self.router_settings.apply_db_row("router_settings", db_values)
|
||||
combined_router_settings: Final = self.router_settings.resolved()
|
||||
if combined_router_settings:
|
||||
self._apply_router_settings(llm_router, combined_router_settings)
|
||||
self._apply_router_settings(
|
||||
llm_router,
|
||||
combined_router_settings,
|
||||
content_policy_owned_by_config=self.router_settings.owned_by_config("content_policy_fallbacks"),
|
||||
content_policy_rules_changed=(
|
||||
combined_router_settings.get("content_policy_fallbacks") != previous_content_policy_rules
|
||||
),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _apply_router_settings(llm_router: Router, router_settings: Mapping[str, object]) -> None:
|
||||
llm_router.apply(ChangeSet(settings={k: v for k, v in router_settings.items() if k != "routing_groups"}))
|
||||
def _apply_router_settings(
|
||||
llm_router: Router,
|
||||
router_settings: Mapping[str, object],
|
||||
*,
|
||||
content_policy_owned_by_config: bool,
|
||||
content_policy_rules_changed: bool,
|
||||
) -> None:
|
||||
llm_router.apply(
|
||||
ChangeSet(
|
||||
settings={
|
||||
k: v for k, v in router_settings.items() if k not in ("routing_groups", "content_policy_fallbacks")
|
||||
}
|
||||
)
|
||||
)
|
||||
if "content_policy_fallbacks" in router_settings and not content_policy_owned_by_config:
|
||||
apply_content_policy_fallbacks(
|
||||
llm_router, router_settings["content_policy_fallbacks"], log_skipped_rules=content_policy_rules_changed
|
||||
)
|
||||
if "routing_groups" not in router_settings:
|
||||
return
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -192,6 +192,11 @@ OptionalPreCallChecks = list[
|
|||
]
|
||||
|
||||
|
||||
FallbackTargets: TypeAlias = Sequence[str | Mapping[str, JsonValue]]
|
||||
ContentPolicyFallbackRule: TypeAlias = Annotated[Mapping[str, FallbackTargets], Field(min_length=1, max_length=1)]
|
||||
ContentPolicyFallbackRules: TypeAlias = Sequence[ContentPolicyFallbackRule]
|
||||
|
||||
|
||||
class UpdateRouterConfig(LiteLLMBaseModel):
|
||||
"""
|
||||
Set of params that you can modify via `router.update_settings()`.
|
||||
|
|
@ -211,6 +216,7 @@ class UpdateRouterConfig(LiteLLMBaseModel):
|
|||
retry_after: float | None = None
|
||||
fallbacks: list[dict] | None = None
|
||||
context_window_fallbacks: list[dict] | None = None
|
||||
content_policy_fallbacks: ContentPolicyFallbackRules | None = None
|
||||
model_group_alias: dict[str, str | dict] | None = Field(default={})
|
||||
enable_tag_filtering: bool | None = None
|
||||
weights: RouterWeights | None = None
|
||||
|
|
|
|||
|
|
@ -7597,6 +7597,7 @@ async def test_generate_key_with_router_settings(monkeypatch):
|
|||
"num_retries": 3,
|
||||
"model_group_retry_policy": {"gpt-4": {"RateLimitErrorRetries": 5}},
|
||||
"weights": {"gpt-4": {"weighted-id": 1}},
|
||||
"content_policy_fallbacks": [{"gpt-4": ["gpt-4o", {"model": "gpt-4o-mini", "temperature": 0}]}],
|
||||
}
|
||||
|
||||
request_data = GenerateKeyRequest(
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ import copy
|
|||
import json
|
||||
from collections.abc import AsyncIterator, Mapping
|
||||
from types import MappingProxyType, SimpleNamespace
|
||||
from typing import Final, get_args
|
||||
from typing import Final, Literal, get_args
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
|
@ -20,6 +20,7 @@ from fastapi import HTTPException
|
|||
from pydantic import JsonValue, TypeAdapter
|
||||
from redis.exceptions import NoPermissionError
|
||||
|
||||
import litellm
|
||||
from litellm import Router
|
||||
from litellm.proxy.config_resolvers.settings_rules import Section
|
||||
from litellm.proxy.management_endpoints.fallback_management_endpoints import (
|
||||
|
|
@ -388,6 +389,74 @@ class TestFallbackWritesSeeTheLatestStoredRules:
|
|||
|
||||
assert stored.stored_fallbacks() == [GATEWAY_RULE, TEAM_RULE]
|
||||
|
||||
async def test_content_policy_writes_keep_the_litellm_settings_rules_for_other_models(
|
||||
self, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "content_policy_fallbacks", [TEAM_RULE])
|
||||
router: Final = _team_router()
|
||||
stored: Final = _StoredRouterSettings(None)
|
||||
with (
|
||||
patch("litellm.proxy.proxy_server.llm_router", router),
|
||||
patch("litellm.proxy.proxy_server.prisma_client", stored.prisma_client),
|
||||
patch("litellm.proxy.proxy_server.proxy_config", stored.proxy_config),
|
||||
patch("litellm.proxy.proxy_server.store_model_in_db", True),
|
||||
):
|
||||
await create_fallback(
|
||||
FallbackCreateRequest(
|
||||
model="gpt-5.4-mini", fallback_models=["team-fallback"], fallback_type="content_policy"
|
||||
),
|
||||
MagicMock(),
|
||||
)
|
||||
after_create: Final = await get_fallback("team-primary", "content_policy", MagicMock())
|
||||
await delete_fallback("gpt-5.4-mini", "content_policy", MagicMock())
|
||||
after_delete: Final = await get_fallback("team-primary", "content_policy", MagicMock())
|
||||
|
||||
assert after_create.fallback_models == ["team-fallback"]
|
||||
assert after_delete.fallback_models == ["team-fallback"]
|
||||
|
||||
async def test_a_content_policy_write_applies_beside_an_invalid_stored_rule(self) -> None:
|
||||
router: Final = _team_router()
|
||||
stored: Final = _StoredRouterSettings({"content_policy_fallbacks": [MALFORMED_GATEWAY_RULE]})
|
||||
with (
|
||||
patch("litellm.proxy.proxy_server.llm_router", router),
|
||||
patch("litellm.proxy.proxy_server.prisma_client", stored.prisma_client),
|
||||
patch("litellm.proxy.proxy_server.proxy_config", stored.proxy_config),
|
||||
patch("litellm.proxy.proxy_server.store_model_in_db", True),
|
||||
):
|
||||
await create_fallback(
|
||||
FallbackCreateRequest(
|
||||
model="team-primary", fallback_models=["team-fallback"], fallback_type="content_policy"
|
||||
),
|
||||
MagicMock(),
|
||||
)
|
||||
after_create: Final = await get_fallback("team-primary", "content_policy", MagicMock())
|
||||
|
||||
assert after_create.fallback_models == ["team-fallback"]
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("fallback_type", "router_setting"),
|
||||
[("general", "fallbacks"), ("content_policy", "content_policy_fallbacks")],
|
||||
)
|
||||
async def test_get_names_the_model_of_a_dict_target(
|
||||
self, fallback_type: Literal["general", "content_policy"], router_setting: str
|
||||
) -> None:
|
||||
router: Final = _team_router()
|
||||
router.update_settings(
|
||||
**{router_setting: [{"team-primary": [{"model": "gpt-5.4-mini", "api_key": "sk-test"}, "team-fallback"]}]}
|
||||
)
|
||||
with patch("litellm.proxy.proxy_server.llm_router", router):
|
||||
response: Final = await get_fallback("team-primary", fallback_type, MagicMock())
|
||||
|
||||
assert response.fallback_models == ["gpt-5.4-mini", "team-fallback"]
|
||||
|
||||
async def test_get_answers_an_error_for_a_string_valued_rule_instead_of_its_characters(self) -> None:
|
||||
router: Final = _team_router()
|
||||
router.update_settings(fallbacks=[{"team-primary": "team-fallback"}])
|
||||
with patch("litellm.proxy.proxy_server.llm_router", router), pytest.raises(HTTPException) as raised:
|
||||
await get_fallback("team-primary", "general", MagicMock())
|
||||
|
||||
assert raised.value.status_code == 500
|
||||
|
||||
async def test_a_create_keeps_a_rule_another_instance_stored_since_this_one_last_read(self) -> None:
|
||||
stored: Final = _StoredRouterSettings({})
|
||||
await get_config_param(stored.prisma_client, "router_settings")
|
||||
|
|
@ -568,6 +637,21 @@ class TestFallbackWritesHonourTheConfigFile:
|
|||
assert stored.row is None
|
||||
assert router.fallbacks == [CONFIG_RULE, CONFIG_WINDOW_RULE]
|
||||
|
||||
async def test_a_content_policy_restate_keeps_the_config_list_without_the_litellm_settings_rules(
|
||||
self, router: Router, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "content_policy_fallbacks", [TEAM_RULE])
|
||||
stored: Final = _StoredRouterSettings(None, config_file={"content_policy_fallbacks": [CONFIG_RULE]})
|
||||
request: Final = FallbackCreateRequest(
|
||||
model="gpt-5.4-mini", fallback_models=["team-fallback"], fallback_type="content_policy"
|
||||
)
|
||||
|
||||
response: Final = await self._create(request, router, stored)
|
||||
|
||||
assert response.fallback_models == ["team-fallback"]
|
||||
assert stored.row is None
|
||||
assert router.content_policy_fallbacks == [CONFIG_RULE]
|
||||
|
||||
async def test_a_create_that_restates_a_config_rule_next_to_an_env_reference_writes_nothing(
|
||||
self, router: Router, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
|
|
|
|||
|
|
@ -10087,9 +10087,12 @@ class TestResolveUserProviderCredentials:
|
|||
assert data["secret_fields"]["user_provider_credentials_user_id"] == "user-a"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_key_router_settings_fallback_loads_per_user_tokens(self, monkeypatch):
|
||||
"""The proxy prefers key/team router_settings.fallbacks over router-level
|
||||
fallbacks (_configured_fallbacks), so discovery must read them too."""
|
||||
@pytest.mark.parametrize("fallback_key", ["fallbacks", "context_window_fallbacks", "content_policy_fallbacks"])
|
||||
async def test_key_router_settings_fallback_loads_per_user_tokens(
|
||||
self, monkeypatch: pytest.MonkeyPatch, fallback_key: str
|
||||
):
|
||||
"""The proxy forwards key/team router_settings fallback lists into the
|
||||
request after discovery, so discovery must read every one of them."""
|
||||
from litellm.proxy.litellm_pre_call_utils import _resolve_user_provider_credentials_for_request
|
||||
|
||||
table = self._env(monkeypatch)
|
||||
|
|
@ -10119,7 +10122,7 @@ class TestResolveUserProviderCredentials:
|
|||
authenticated_user_id="user-a",
|
||||
team_id=None,
|
||||
llm_router=router,
|
||||
router_settings={"fallbacks": [{"gpt-4o": [{"model": "copilot-chat"}]}]},
|
||||
router_settings={fallback_key: [{"gpt-4o": [{"model": "copilot-chat"}]}]},
|
||||
)
|
||||
table.find_many.assert_awaited()
|
||||
assert data["secret_fields"]["user_provider_credentials_user_id"] == "user-a"
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import subprocess
|
|||
import sys
|
||||
import time
|
||||
import types
|
||||
from collections.abc import Callable
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
|
@ -5583,6 +5584,200 @@ async def test_valid_db_routing_groups_still_replace_router_groups():
|
|||
assert router._get_routing_context("m2", None)[0] == "least-busy"
|
||||
|
||||
|
||||
def _content_policy_router(**router_kwargs: object) -> litellm.Router:
|
||||
return litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "primary",
|
||||
"litellm_params": {"model": "openai/gpt-5.4-mini", "api_key": "fake", "mock_response": "primary answered"},
|
||||
},
|
||||
{
|
||||
"model_name": "secondary",
|
||||
"litellm_params": {"model": "openai/gpt-5.4-mini", "api_key": "fake", "mock_response": "secondary answered"},
|
||||
},
|
||||
{
|
||||
"model_name": "backup",
|
||||
"litellm_params": {"model": "openai/gpt-5.4-nano", "api_key": "fake", "mock_response": "backup answered"},
|
||||
},
|
||||
],
|
||||
num_retries=0,
|
||||
**router_kwargs,
|
||||
)
|
||||
|
||||
|
||||
def _router_settings_row(content_policy_fallbacks: object) -> MagicMock:
|
||||
row: Final = MagicMock()
|
||||
row.param_value = {"allowed_fails": 7, "content_policy_fallbacks": content_policy_fallbacks}
|
||||
prisma: Final = MagicMock()
|
||||
prisma.db.litellm_config.find_first = AsyncMock(return_value=row)
|
||||
return prisma
|
||||
|
||||
|
||||
async def _answer_after_content_policy_refusal(router: litellm.Router, model: str = "primary") -> str | None:
|
||||
response: Final = await router.acompletion(
|
||||
model=model, messages=[{"role": "user", "content": "hi"}], mock_testing_content_policy_fallbacks=True
|
||||
)
|
||||
return response.choices[0].message.content
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("stored_target", ["backup", {"model": "backup"}])
|
||||
async def test_db_content_policy_fallbacks_reach_the_router(stored_target: str | dict[str, str]):
|
||||
router: Final = _content_policy_router()
|
||||
|
||||
await proxy_server_module.ProxyConfig()._add_router_settings_from_db_config(
|
||||
llm_router=router, prisma_client=_router_settings_row([{"primary": [stored_target]}])
|
||||
)
|
||||
|
||||
assert await _answer_after_content_policy_refusal(router) == "backup answered"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("litellm_settings_rules", "stored_rules", "refused_model"),
|
||||
[
|
||||
([{"secondary": ["backup"]}], [{"primary": ["backup"]}], "secondary"),
|
||||
([{"*": ["secondary"]}], [{"*": ["backup"]}], "primary"),
|
||||
([{"primary": ["backup"]}], [], "primary"),
|
||||
],
|
||||
)
|
||||
async def test_db_content_policy_fallbacks_layer_over_the_litellm_settings_rules(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
litellm_settings_rules: list[dict[str, list[str]]],
|
||||
stored_rules: list[dict[str, list[str]]],
|
||||
refused_model: str,
|
||||
):
|
||||
monkeypatch.setattr(litellm, "content_policy_fallbacks", litellm_settings_rules)
|
||||
router: Final = _content_policy_router()
|
||||
|
||||
await proxy_server_module.ProxyConfig()._add_router_settings_from_db_config(
|
||||
llm_router=router, prisma_client=_router_settings_row(stored_rules)
|
||||
)
|
||||
|
||||
assert await _answer_after_content_policy_refusal(router, refused_model) == "backup answered"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"invalid_rule",
|
||||
[{"secondary": "backup"}, ["backup"], dict[str, list[str]](), {"secondary": ["backup"], "primary": ["secondary"]}],
|
||||
)
|
||||
async def test_an_invalid_db_content_policy_rule_skips_only_itself(monkeypatch: pytest.MonkeyPatch, invalid_rule: object):
|
||||
monkeypatch.setattr(litellm, "content_policy_fallbacks", [{"primary": ["backup"]}])
|
||||
router: Final = _content_policy_router()
|
||||
|
||||
await proxy_server_module.ProxyConfig()._add_router_settings_from_db_config(
|
||||
llm_router=router, prisma_client=_router_settings_row([invalid_rule, {"secondary": ["backup"]}])
|
||||
)
|
||||
|
||||
assert router.allowed_fails == 7
|
||||
assert await _answer_after_content_policy_refusal(router) == "backup answered"
|
||||
assert await _answer_after_content_policy_refusal(router, "secondary") == "backup answered"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_db_content_policy_value_that_is_not_a_list_keeps_the_litellm_settings_rules(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
monkeypatch.setattr(litellm, "content_policy_fallbacks", [{"primary": ["backup"]}])
|
||||
router: Final = _content_policy_router()
|
||||
|
||||
await proxy_server_module.ProxyConfig()._add_router_settings_from_db_config(
|
||||
llm_router=router, prisma_client=_router_settings_row("oops")
|
||||
)
|
||||
|
||||
assert router.allowed_fails == 7
|
||||
assert await _answer_after_content_policy_refusal(router) == "backup answered"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_db_sync_keeps_a_litellm_settings_rule_with_a_target_router_init_accepted(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
monkeypatch.setattr(litellm, "content_policy_fallbacks", [{"primary": ["backup", 5]}])
|
||||
router: Final = _content_policy_router()
|
||||
|
||||
await proxy_server_module.ProxyConfig()._add_router_settings_from_db_config(
|
||||
llm_router=router, prisma_client=_router_settings_row([{"secondary": ["backup"]}])
|
||||
)
|
||||
|
||||
assert await _answer_after_content_policy_refusal(router) == "backup answered"
|
||||
assert await _answer_after_content_policy_refusal(router, "secondary") == "backup answered"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_invalid_db_content_policy_rule_is_logged_once_per_change_of_the_rules(
|
||||
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
|
||||
):
|
||||
monkeypatch.setattr(litellm, "content_policy_fallbacks", [])
|
||||
router: Final = _content_policy_router()
|
||||
proxy_config: Final = proxy_server_module.ProxyConfig()
|
||||
stored_rules: Final = [{"secondary": "backup"}, {"primary": ["backup"]}]
|
||||
changed_rules: Final = [{"secondary": "backup"}, {"secondary": ["backup"]}]
|
||||
|
||||
with caplog.at_level(logging.ERROR, logger="LiteLLM Proxy"):
|
||||
await proxy_config._add_router_settings_from_db_config(
|
||||
llm_router=router, prisma_client=_router_settings_row(stored_rules)
|
||||
)
|
||||
await proxy_config._add_router_settings_from_db_config(
|
||||
llm_router=router, prisma_client=_router_settings_row(stored_rules)
|
||||
)
|
||||
await proxy_config._add_router_settings_from_db_config(
|
||||
llm_router=router, prisma_client=_router_settings_row(changed_rules)
|
||||
)
|
||||
|
||||
notices: Final = [
|
||||
record for record in caplog.records if "Ignoring invalid content_policy_fallbacks" in record.getMessage()
|
||||
]
|
||||
assert len(notices) == 2
|
||||
assert await _answer_after_content_policy_refusal(router, "secondary") == "backup answered"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_db_sync_logs_a_new_invalid_content_policy_rule_when_no_valid_rule_changes(
|
||||
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
|
||||
):
|
||||
monkeypatch.setattr(litellm, "content_policy_fallbacks", [])
|
||||
router: Final = _content_policy_router()
|
||||
proxy_config: Final = proxy_server_module.ProxyConfig()
|
||||
|
||||
with caplog.at_level(logging.ERROR, logger="LiteLLM Proxy"):
|
||||
await proxy_config._add_router_settings_from_db_config(
|
||||
llm_router=router, prisma_client=_router_settings_row([{"secondary": "backup"}])
|
||||
)
|
||||
await proxy_config._add_router_settings_from_db_config(
|
||||
llm_router=router, prisma_client=_router_settings_row([{"secondary": "backup"}, {"primary": "backup"}])
|
||||
)
|
||||
|
||||
notices: Final = [
|
||||
record.getMessage()
|
||||
for record in caplog.records
|
||||
if "Ignoring invalid content_policy_fallbacks" in record.getMessage()
|
||||
]
|
||||
assert len(notices) == 3
|
||||
assert "{'primary': 'backup'}" in notices[-1]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_config_file_content_policy_fallbacks_stay_the_only_rules_after_a_db_sync(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
monkeypatch.setattr(litellm, "content_policy_fallbacks", [{"secondary": ["backup"]}])
|
||||
file_rules: Final = [{"primary": ["backup"]}]
|
||||
router: Final = _content_policy_router(content_policy_fallbacks=file_rules)
|
||||
proxy_config: Final = proxy_server_module.ProxyConfig()
|
||||
proxy_config.router_settings.load_yaml({"content_policy_fallbacks": file_rules})
|
||||
|
||||
await proxy_config._add_router_settings_from_db_config(
|
||||
llm_router=router, prisma_client=_router_settings_row([{"secondary": ["backup"]}])
|
||||
)
|
||||
|
||||
assert router.allowed_fails == 7
|
||||
assert await _answer_after_content_policy_refusal(router) == "backup answered"
|
||||
with pytest.raises(litellm.ContentPolicyViolationError):
|
||||
await _answer_after_content_policy_refusal(router, "secondary")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_router_settings_from_db_config_empty_db_lists_do_not_clobber_config_fallbacks():
|
||||
"""
|
||||
|
|
@ -5591,6 +5786,7 @@ async def test_add_router_settings_from_db_config_empty_db_lists_do_not_clobber_
|
|||
empty lists are "no value" and must not clobber config.yaml fallbacks,
|
||||
matching _deep_merge_dicts semantics. Non-empty DB lists still win.
|
||||
"""
|
||||
from collections import ChainMap
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
from litellm.proxy.proxy_server import ProxyConfig
|
||||
|
|
@ -5625,10 +5821,10 @@ async def test_add_router_settings_from_db_config_empty_db_lists_do_not_clobber_
|
|||
prisma_client=mock_prisma_client,
|
||||
)
|
||||
|
||||
combined_settings = mock_router.update_settings.call_args.kwargs
|
||||
combined_settings = ChainMap(*(call.kwargs for call in reversed(mock_router.update_settings.call_args_list)))
|
||||
assert combined_settings["fallbacks"] == [{"gpt-oss-120b": ["granite-4-h-small"]}]
|
||||
assert combined_settings["context_window_fallbacks"] == [{"gpt-oss-120b": ["granite-4-h-small"]}]
|
||||
assert combined_settings["content_policy_fallbacks"] == [{"gpt-oss-120b": ["granite-4-h-small"]}]
|
||||
assert combined_settings.get("content_policy_fallbacks") != [{"gpt-oss-120b": ["other-model"]}]
|
||||
assert combined_settings["num_retries"] == 3
|
||||
|
||||
|
||||
|
|
@ -9997,6 +10193,34 @@ def test_update_config_accepts_disjoint_routing_groups(_update_config_setup):
|
|||
restore()
|
||||
|
||||
|
||||
def test_update_config_validates_content_policy_fallbacks_like_fallbacks(
|
||||
_update_config_setup: Callable[..., tuple[TestClient, _FakePrismaClient, Callable[[], None]]],
|
||||
):
|
||||
client, prisma, restore = _update_config_setup(initial_rows={"router_settings": {"num_retries": 2}})
|
||||
rule: Final = [{"gpt-5.4-mini": ["gpt-5.4-nano", {"model": "gpt-5.4-nano", "temperature": 1}]}]
|
||||
malformed: Final = (
|
||||
"oops",
|
||||
5,
|
||||
[1, 2],
|
||||
[{"gpt-5.4-mini": "gpt-5.4-nano"}],
|
||||
[{"gpt-5.4-mini": ["gpt-5.4-nano", 7]}],
|
||||
[dict[str, list[str]]()],
|
||||
[{"gpt-5.4-mini": ["gpt-5.4-nano"], "gpt-5.4": ["gpt-5.4-nano"]}],
|
||||
)
|
||||
try:
|
||||
rejected: Final = tuple(
|
||||
client.post("/config/update", json={"router_settings": {"content_policy_fallbacks": value}}).status_code
|
||||
for value in malformed
|
||||
)
|
||||
assert rejected == (422,) * len(malformed)
|
||||
assert prisma.db.litellm_config.rows["router_settings"] == {"num_retries": 2}
|
||||
accepted: Final = client.post("/config/update", json={"router_settings": {"content_policy_fallbacks": rule}})
|
||||
assert accepted.status_code == 200
|
||||
assert prisma.db.litellm_config.rows["router_settings"] == {"num_retries": 2, "content_policy_fallbacks": rule}
|
||||
finally:
|
||||
restore()
|
||||
|
||||
|
||||
def test_update_config_env_var_round_trip_not_double_encrypted(_update_config_setup, monkeypatch):
|
||||
"""Endpoint-level regression for the /config/update double-encryption bug.
|
||||
|
||||
|
|
|
|||
|
|
@ -2730,6 +2730,39 @@ def test_update_settings_model_group_alias_drops_cached_group_info():
|
|||
assert after.input_cost_per_token is not None and after.input_cost_per_token > 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"fallback_kind, mock_testing_param",
|
||||
[
|
||||
("fallbacks", "mock_testing_fallbacks"),
|
||||
("context_window_fallbacks", "mock_testing_context_fallbacks"),
|
||||
("content_policy_fallbacks", "mock_testing_content_policy_fallbacks"),
|
||||
],
|
||||
)
|
||||
async def test_update_settings_fallback_rule_of_every_kind_is_used_on_its_error(
|
||||
fallback_kind: str, mock_testing_param: str
|
||||
):
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "primary",
|
||||
"litellm_params": {"model": "gpt-5.4-mini", "api_key": "fake", "mock_response": "primary answered"},
|
||||
},
|
||||
{
|
||||
"model_name": "backup",
|
||||
"litellm_params": {"model": "gpt-5.4-nano", "api_key": "fake", "mock_response": "backup answered"},
|
||||
},
|
||||
],
|
||||
num_retries=0,
|
||||
)
|
||||
|
||||
router.update_settings(**{fallback_kind: [{"primary": ["backup"]}]})
|
||||
|
||||
response: Final = await router.acompletion(
|
||||
model="primary", messages=[{"role": "user", "content": "hi"}], **{mock_testing_param: True}
|
||||
)
|
||||
assert response.choices[0].message.content == "backup answered"
|
||||
|
||||
|
||||
_PAID_INPUT_COST_PER_TOKEN: Final = 3e-06
|
||||
_PAID_OUTPUT_COST_PER_TOKEN: Final = 1.5e-05
|
||||
|
||||
|
|
|
|||
6
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
6
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -48834,6 +48834,12 @@ export interface components {
|
|||
UpdateRouterConfig: {
|
||||
/** Allowed Fails */
|
||||
allowed_fails?: number | null;
|
||||
/** Content Policy Fallbacks */
|
||||
content_policy_fallbacks?: {
|
||||
[key: string]: (string | {
|
||||
[key: string]: components["schemas"]["JsonValue"];
|
||||
})[];
|
||||
}[] | null;
|
||||
/** Context Window Fallbacks */
|
||||
context_window_fallbacks?: {
|
||||
[key: string]: unknown;
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue