diff --git a/litellm/proxy/management_endpoints/fallback_management_endpoints.py b/litellm/proxy/management_endpoints/fallback_management_endpoints.py index 6a9a30a7689..94cb6dc8efe 100644 --- a/litellm/proxy/management_endpoints/fallback_management_endpoints.py +++ b/litellm/proxy/management_endpoints/fallback_management_endpoints.py @@ -20,7 +20,7 @@ from litellm._logging import verbose_proxy_logger from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.model_checks import get_all_fallbacks from litellm.proxy.auth.user_api_key_auth import user_api_key_auth -from litellm.proxy.utils import PrismaClient, evict_config_param, invalidate_config_param +from litellm.proxy.utils import PrismaClient, invalidate_config_param if TYPE_CHECKING: from fastapi import APIRouter, Depends, HTTPException, status @@ -48,16 +48,18 @@ FallbackRule = dict[str, list[str]] StoredFallback = Annotated[FallbackRule | dict[str, object] | str, Field(union_mode="left_to_right")] _STORED_FALLBACKS: Final = TypeAdapter(list[StoredFallback]) _JSON_RULES: Final = TypeAdapter(list[JsonValue]) +_STORED_ROUTER_SETTINGS: Final = TypeAdapter(dict[str, JsonValue]) def _rule_covers(entry: StoredFallback, model: str) -> bool: return isinstance(entry, dict) and model in entry -async def _router_settings_fresh_from_db(proxy_config: "ProxyConfig") -> dict[str, object]: - await evict_config_param(ROUTER_SETTINGS_PARAM) - config: Final = await proxy_config.get_config() - return config.get(ROUTER_SETTINGS_PARAM, {}) +async def _stored_router_settings(prisma_client: PrismaClient) -> Mapping[str, JsonValue]: + row: Final = await ConfigRepository(prisma_client, use_writer=True).get_param(ROUTER_SETTINGS_PARAM) + if row is None or row.param_value is None: + return {} + return _STORED_ROUTER_SETTINGS.validate_python(row.param_value) def _reject_config_owned_rules(proxy_config: "ProxyConfig", fallback_key: str, rules: Sequence[StoredFallback]) -> None: @@ -162,7 +164,7 @@ async def create_fallback( }, ) - router_settings: Final = await _router_settings_fresh_from_db(proxy_config) + router_settings: Final = await _stored_router_settings(prisma_client) # Get the appropriate fallback list based on type fallback_key = "fallbacks" @@ -189,10 +191,7 @@ async def create_fallback( _reject_config_owned_rules(proxy_config, fallback_key, existing_fallbacks) - # Update router settings - router_settings[fallback_key] = existing_fallbacks - - await _persist_router_settings(prisma_client, router_settings) + await _persist_router_settings(prisma_client, {**router_settings, fallback_key: existing_fallbacks}) # Update the in-memory router configuration setattr(llm_router, fallback_key, existing_fallbacks) @@ -320,7 +319,7 @@ async def delete_fallback( }, ) - router_settings: Final = await _router_settings_fresh_from_db(proxy_config) + router_settings: Final = await _stored_router_settings(prisma_client) # Get the appropriate fallback list based on type fallback_key = "fallbacks" @@ -342,10 +341,7 @@ async def delete_fallback( _reject_config_owned_rules(proxy_config, fallback_key, updated_fallbacks) - # Update router settings - router_settings[fallback_key] = updated_fallbacks - - await _persist_router_settings(prisma_client, router_settings) + await _persist_router_settings(prisma_client, {**router_settings, fallback_key: updated_fallbacks}) # Update the in-memory router configuration setattr(llm_router, fallback_key, updated_fallbacks) diff --git a/tests/unit/proxy/test_fallback_management_endpoints.py b/tests/unit/proxy/test_fallback_management_endpoints.py index 530d22f02be..e024d1e84c3 100644 --- a/tests/unit/proxy/test_fallback_management_endpoints.py +++ b/tests/unit/proxy/test_fallback_management_endpoints.py @@ -18,6 +18,7 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest from fastapi import HTTPException from pydantic import JsonValue, TypeAdapter +from redis.exceptions import NoPermissionError from litellm import Router from litellm.proxy.management_endpoints.fallback_management_endpoints import ( @@ -29,7 +30,7 @@ from litellm.proxy.management_endpoints.fallback_management_endpoints import ( get_fallback, ) from litellm.proxy.proxy_server import ProxyConfig -from litellm.proxy.utils import evict_config_param, get_config_param +from litellm.proxy.utils import evict_config_param, get_config_param, litellm_config_cache from litellm.types.router import DeploymentTypedDict, LiteLLMParamsTypedDict @@ -168,14 +169,14 @@ class TestCreateFallbackForTeamScopedModels: @pytest.fixture def prisma_client(self) -> MagicMock: client: Final = MagicMock() + client.writer_db = client.db + client.db.litellm_config.find_unique = AsyncMock(return_value=None) client.db.litellm_config.upsert = AsyncMock() return client @pytest.fixture def proxy_config(self) -> MagicMock: - config: Final = MagicMock() - config.get_config = AsyncMock(return_value={"router_settings": {}}) - return config + return MagicMock() async def _create( self, request: FallbackCreateRequest, router: Router, prisma_client: MagicMock, proxy_config: MagicMock @@ -259,8 +260,11 @@ class _StoredRouterSettings: def __init__(self, router_settings: RouterSettings | None, config_file: RouterSettings | None = None) -> None: self.row: SimpleNamespace | None = None if router_settings is None else _config_row(router_settings) + self.config_file: Final = config_file or {} self.prisma_client: Final = MagicMock() self.prisma_client.get_generic_data = AsyncMock(side_effect=lambda **_: self.row) + self.prisma_client.writer_db = self.prisma_client.db + self.prisma_client.db.litellm_config.find_unique = AsyncMock(side_effect=lambda **_: copy.deepcopy(self.row)) self.prisma_client.db.litellm_config.upsert = AsyncMock(side_effect=self._upsert) self.proxy_config: Final = MagicMock() if config_file is None else ProxyConfig() if config_file is not None: @@ -272,7 +276,8 @@ class _StoredRouterSettings: async def _get_config(self) -> dict[str, RouterSettings]: row: Final = await get_config_param(self.prisma_client, "router_settings") - return {"router_settings": copy.deepcopy(row.param_value) if row is not None else {}} + stored: Final = copy.deepcopy(row.param_value) if row is not None else {} + return {"router_settings": {**stored, **self.config_file}} def written_by_another_instance(self, router_settings: RouterSettings) -> None: self.row = _config_row(router_settings) @@ -295,6 +300,20 @@ NON_STANDARD_RULES: Final = [ ] +class _RedisThatRefusesDel: + def __init__(self) -> None: + self.value: object = None + + async def async_get_cache(self, key: str, parent_otel_span: object = None) -> object: + return self.value + + async def async_set_cache(self, key: str, value: object, **_: object) -> None: + self.value = value + + async def async_delete_cache(self, key: str) -> None: + raise NoPermissionError("this user has no permissions to run the 'del' command") + + class TestFallbackWritesSeeTheLatestStoredRules: """A write reads the rules the database holds now and leaves no stale copy in the config cache behind""" @@ -331,6 +350,17 @@ class TestFallbackWritesSeeTheLatestStoredRules: assert stored.stored_fallbacks() == [TEAM_RULE, GATEWAY_RULE] assert await stored.cached_fallbacks() == [TEAM_RULE, GATEWAY_RULE] + async def test_a_second_create_keeps_the_first_rule_when_redis_refuses_to_evict_the_cached_settings(self) -> None: + stored: Final = _StoredRouterSettings(None) + + with patch.object(litellm_config_cache, "redis_cache", _RedisThatRefusesDel()): + await self._create(FallbackCreateRequest(model="team-primary", fallback_models=["team-fallback"]), stored) + await self._create(FallbackCreateRequest(model="gpt-5.4-mini", fallback_models=["team-fallback"]), stored) + await self._delete("team-primary", stored) + await self._create(FallbackCreateRequest(model="team-primary", fallback_models=["team-fallback"]), stored) + + assert stored.stored_fallbacks() == [GATEWAY_RULE, TEAM_RULE] + async def test_a_create_keeps_a_rule_another_instance_stored_since_this_one_last_read(self) -> None: stored: Final = _StoredRouterSettings({}) await get_config_param(stored.prisma_client, "router_settings") @@ -380,6 +410,14 @@ class TestFallbackWritesSeeTheLatestStoredRules: assert stored.stored_fallbacks() == [GATEWAY_RULE] + async def test_a_create_treats_a_null_stored_row_as_empty(self) -> None: + stored: Final = _StoredRouterSettings(None) + stored.row = SimpleNamespace(param_name="router_settings", param_value=None) + + await self._create(FallbackCreateRequest(model="gpt-5.4-mini", fallback_models=["team-fallback"]), stored) + + assert stored.stored_fallbacks() == [GATEWAY_RULE] + CONFIG_RULE: Final = {"gpt-5.4-mini": ["team-fallback"]} CONFIG_WINDOW_RULE: Final = {"team-primary": ["gpt-5.4-mini"]} @@ -462,6 +500,20 @@ class TestFallbackWritesHonourTheConfigFile: assert response.fallback_models == ["team-fallback"] assert stored.stored_fallbacks() == [CONFIG_RULE] + async def test_a_write_stores_only_the_database_rules_never_the_config_files_router_settings( + self, router: Router + ) -> None: + stored: Final = _StoredRouterSettings( + {"fallbacks": [TEAM_RULE]}, config_file={"routing_strategy": "latency-based-routing"} + ) + + await self._create(FallbackCreateRequest(model="gpt-5.4-mini", fallback_models=["team-fallback"]), router, stored) + assert stored.row is not None + assert stored.row.param_value == {"fallbacks": [TEAM_RULE, GATEWAY_RULE]} + + await self._delete("team-primary", router, stored) + assert stored.row.param_value == {"fallbacks": [GATEWAY_RULE]} + async def test_a_sibling_list_the_config_file_does_not_set_stays_writable( self, router: Router, stored: _StoredRouterSettings ) -> None: @@ -496,6 +548,8 @@ class TestCreateFallback: def mock_prisma_client(self): """Create a mock prisma client""" client = MagicMock() + client.writer_db = client.db + client.db.litellm_config.find_unique = AsyncMock(return_value=None) client.db.litellm_config.upsert = AsyncMock() client.jsonify_object = lambda x: x return client @@ -503,9 +557,7 @@ class TestCreateFallback: @pytest.fixture def mock_proxy_config(self): """Create a mock proxy config""" - config = MagicMock() - config.get_config = AsyncMock(return_value={"router_settings": {}}) - return config + return MagicMock() @pytest.fixture def mock_user_api_key_dict(self): @@ -803,6 +855,10 @@ class TestDeleteFallback: def mock_prisma_client(self): """Create a mock prisma client""" client = MagicMock() + client.writer_db = client.db + client.db.litellm_config.find_unique = AsyncMock( + return_value=_config_row({"fallbacks": [{"gpt-3.5-turbo": ["gpt-4", "claude-3-haiku"]}]}) + ) client.db.litellm_config.upsert = AsyncMock() client.jsonify_object = lambda x: x return client @@ -810,15 +866,7 @@ class TestDeleteFallback: @pytest.fixture def mock_proxy_config(self): """Create a mock proxy config""" - config = MagicMock() - config.get_config = AsyncMock( - return_value={ - "router_settings": { - "fallbacks": [{"gpt-3.5-turbo": ["gpt-4", "claude-3-haiku"]}] - } - } - ) - return config + return MagicMock() @pytest.fixture def mock_user_api_key_dict(self):