mirror of
https://github.com/BerriAI/litellm.git
synced 2026-08-28 05:25:59 +00:00
fix(proxy): publish router_settings changes so peer pods apply them on resync
add_deployment already reapplies DB router settings through _update_llm_router, so gating router_settings out of the pub/sub publish set left the push path covering less than the resync actually applies
This commit is contained in:
parent
a8018f7500
commit
77e490a695
3 changed files with 43 additions and 4 deletions
|
|
@ -60,6 +60,7 @@ _CONFIG_SYNCED_TABLE_NAMES: frozenset[str] = frozenset(
|
|||
_RESYNC_APPLIED_CONFIG_PARAM_NAMES: frozenset[str] = frozenset(
|
||||
{
|
||||
"general_settings",
|
||||
"router_settings",
|
||||
"litellm_settings",
|
||||
"model_cost_map_reload_config",
|
||||
"anthropic_beta_headers_reload_config",
|
||||
|
|
|
|||
|
|
@ -56,10 +56,11 @@ _EXPECTED_RESYNC_APPLIED_CONFIG_PARAM_NAMES = frozenset(
|
|||
"general_settings",
|
||||
"litellm_settings",
|
||||
"model_cost_map_reload_config",
|
||||
"router_settings",
|
||||
}
|
||||
)
|
||||
|
||||
_STARTUP_ONLY_CONFIG_PARAM_NAMES = ("environment_variables", "router_settings")
|
||||
_STARTUP_ONLY_CONFIG_PARAM_NAMES = ("environment_variables",)
|
||||
|
||||
|
||||
class _RecordingRedisClient(Redis):
|
||||
|
|
@ -716,13 +717,14 @@ async def _publish_calls_for_invalidated_param(param_name: str) -> List[Tuple[st
|
|||
return client.published
|
||||
|
||||
|
||||
async def test_invalidate_config_param_publishes_params_a_resync_applies() -> None:
|
||||
published = await _publish_calls_for_invalidated_param("general_settings")
|
||||
@pytest.mark.parametrize("param_name", sorted(_EXPECTED_RESYNC_APPLIED_CONFIG_PARAM_NAMES))
|
||||
async def test_invalidate_config_param_publishes_params_a_resync_applies(param_name: str) -> None:
|
||||
published = await _publish_calls_for_invalidated_param(param_name)
|
||||
|
||||
assert len(published) == 1
|
||||
channel, message = published[0]
|
||||
assert channel == CONFIG_SYNC_CHANNEL
|
||||
assert json.loads(message) == {"object_type": "general_settings"}
|
||||
assert json.loads(message) == {"object_type": param_name}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("param_name", _STARTUP_ONLY_CONFIG_PARAM_NAMES)
|
||||
|
|
|
|||
|
|
@ -2058,6 +2058,42 @@ async def test_ProxyConfig__add_router_settings_from_db_config_none_router_noop(
|
|||
await pc._add_router_settings_from_db_config() # type: ignore[call-arg]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ProxyConfig.add_deployment
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ProxyConfig_add_deployment_applies_db_router_settings(monkeypatch):
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
pc = ProxyConfig()
|
||||
fake_router = MagicMock()
|
||||
fake_router.get_model_list = MagicMock(return_value=[])
|
||||
fake_prisma = MagicMock()
|
||||
fake_prisma.db.litellm_config.find_first = AsyncMock(
|
||||
return_value=SimpleNamespace(param_value={"routing_strategy": "latency-based-routing"})
|
||||
)
|
||||
|
||||
async def fake_get_config(*args, **kwargs):
|
||||
return {}
|
||||
|
||||
monkeypatch.setattr(pc, "get_config", fake_get_config)
|
||||
monkeypatch.setattr(pc, "_get_models_from_db", AsyncMock(return_value=[]))
|
||||
monkeypatch.setattr(pc, "_init_non_llm_objects_in_db", AsyncMock())
|
||||
monkeypatch.setattr(proxy_server, "prefetch_config_params", AsyncMock())
|
||||
monkeypatch.setattr(proxy_server, "get_config_param", AsyncMock(return_value=None))
|
||||
monkeypatch.setattr(proxy_server, "llm_router", fake_router)
|
||||
monkeypatch.setattr(proxy_server, "master_key", "sk-master")
|
||||
monkeypatch.setattr(proxy_server, "prisma_client", fake_prisma)
|
||||
monkeypatch.setattr(proxy_server, "general_settings", {})
|
||||
monkeypatch.setattr(proxy_server, "proxy_config", pc)
|
||||
|
||||
await pc.add_deployment(prisma_client=fake_prisma, proxy_logging_obj=MagicMock())
|
||||
|
||||
fake_router.update_settings.assert_called_once_with(routing_strategy="latency-based-routing")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ProxyConfig._add_general_settings_from_db_config
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue