scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-09 22:31:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 86

Merge 34190b81a0 into b618c7ad86

Browse source
This commit is contained in:
woshiliangwenfeng 2026-09-07 13:49:52 -07:00 • committed by GitHub
commit ba82f30610
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 212 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
litellm/proxy/management_endpoints/router_settings_endpoints.py
View file

@ -107,7 +107,31 @@ async def get_router_settings(
current_values[field.field_name] = value
# Merge with config values (config takes precedence)
current_values.update(router_settings_from_config)
# Fix: Special handling for routing_groups to prevent overwriting
# llm_router memory values with potentially empty config values
for key, value in router_settings_from_config.items():
if key == "routing_groups":
# If llm_router has routing_groups and config value is empty/missing,
# keep the llm_router value
if key in current_values and current_values[key]:
# llm_router has value, check if config value is meaningful
if not value:
verbose_proxy_logger.debug(
f"Skipping empty routing_groups from config, "
f"keeping llm_router value: {len(current_values[key])} groups"
)
continue
# Both have values, merge them (config takes precedence for same group)
existing_groups = {g["group_name"]: g for g in current_values[key]}
new_groups = {g["group_name"]: g for g in value}
existing_groups.update(new_groups)
current_values[key] = list(existing_groups.values())
continue
# llm_router doesn't have value, use config value (even if empty)
current_values[key] = value if value else []
else:
# For other fields, config takes precedence
current_values[key] = value
# Update field values with current values
for field in router_fields:

187
tests/test_litellm/proxy/management_endpoints/test_router_settings_endpoints.py
View file

@ -119,3 +119,190 @@ class TestRouterSettingsEndpoints:
rg_field = next(f for f in response.fields if f.field_name == "routing_groups")
assert rg_field.field_value == groups
@pytest.mark.asyncio
async def test_get_router_settings_empty_config_routing_groups_not_overwritten(
self, monkeypatch
):
"""When config returns empty routing_groups, live router value is preserved."""
live_groups = [
{
"group_name": "live-group",
"models": ["gpt-4"],
"routing_strategy": "cost-based-routing",
}
]
llm_router = Router(
model_list=[
{
"model_name": "gpt-4",
"litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-x"},
}
],
routing_groups=live_groups,
)
monkeypatch.setattr(proxy_server, "llm_router", llm_router)
async def fake_get_config(self, config_file_path=None):
return {"router_settings": {"routing_groups": []}}
monkeypatch.setattr(
proxy_server.ProxyConfig, "get_config", fake_get_config, raising=True
)
admin_user = UserAPIKeyAuth(
user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-x"
)
response = await get_router_settings(user_api_key_dict=admin_user)
assert response.current_values.get("routing_groups") == live_groups
@pytest.mark.asyncio
async def test_get_router_settings_merge_routing_groups_config_wins(
self, monkeypatch
):
"""When both live router and config have routing_groups, config values win for same group_name."""
live_groups = [
{
"group_name": "shared-group",
"models": ["gpt-4"],
"routing_strategy": "cost-based-routing",
},
{
"group_name": "live-only-group",
"models": ["gpt-3.5"],
"routing_strategy": "simple-shuffle",
},
]
config_groups = [
{
"group_name": "shared-group",
"models": ["gpt-4o"],
"routing_strategy": "latency-based-routing",
},
{
"group_name": "config-only-group",
"models": ["claude-3"],
"routing_strategy": "least-busy",
},
]
llm_router = Router(
model_list=[
{
"model_name": "gpt-4",
"litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-x"},
},
{
"model_name": "gpt-3.5",
"litellm_params": {"model": "openai/gpt-3.5-turbo", "api_key": "sk-x"},
},
{
"model_name": "gpt-4o",
"litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-x"},
},
{
"model_name": "claude-3",
"litellm_params": {"model": "anthropic/claude-3", "api_key": "sk-x"},
},
],
routing_groups=live_groups,
)
monkeypatch.setattr(proxy_server, "llm_router", llm_router)
async def fake_get_config(self, config_file_path=None):
return {"router_settings": {"routing_groups": config_groups}}
monkeypatch.setattr(
proxy_server.ProxyConfig, "get_config", fake_get_config, raising=True
)
admin_user = UserAPIKeyAuth(
user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-x"
)
response = await get_router_settings(user_api_key_dict=admin_user)
result_groups = response.current_values.get("routing_groups", [])
result_by_name = {g["group_name"]: g for g in result_groups}
# shared-group should have config values
assert result_by_name["shared-group"]["routing_strategy"] == "latency-based-routing"
assert result_by_name["shared-group"]["models"] == ["gpt-4o"]
# live-only-group should be preserved
assert "live-only-group" in result_by_name
assert result_by_name["live-only-group"]["routing_strategy"] == "simple-shuffle"
# config-only-group should be added
assert "config-only-group" in result_by_name
assert result_by_name["config-only-group"]["routing_strategy"] == "least-busy"
@pytest.mark.asyncio
async def test_get_router_settings_fallback_to_config_routing_groups(
self, monkeypatch
):
"""When live router has no routing_groups, config value is used."""
llm_router = Router(
model_list=[
{
"model_name": "gpt-4",
"litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-x"},
}
],
)
monkeypatch.setattr(proxy_server, "llm_router", llm_router)
config_groups = [
{
"group_name": "config-group",
"models": ["gpt-4"],
"routing_strategy": "cost-based-routing",
}
]
async def fake_get_config(self, config_file_path=None):
return {"router_settings": {"routing_groups": config_groups}}
monkeypatch.setattr(
proxy_server.ProxyConfig, "get_config", fake_get_config, raising=True
)
admin_user = UserAPIKeyAuth(
user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-x"
)
response = await get_router_settings(user_api_key_dict=admin_user)
assert response.current_values.get("routing_groups") == config_groups
@pytest.mark.asyncio
async def test_get_router_settings_other_fields_overwritten_by_config(
self, monkeypatch
):
"""Non-routing_groups fields should still be overwritten by config values."""
llm_router = Router(
model_list=[
{
"model_name": "gpt-4",
"litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-x"},
}
],
num_retries=5,
)
monkeypatch.setattr(proxy_server, "llm_router", llm_router)
async def fake_get_config(self, config_file_path=None):
return {"router_settings": {"num_retries": 10}}
monkeypatch.setattr(
proxy_server.ProxyConfig, "get_config", fake_get_config, raising=True
)
admin_user = UserAPIKeyAuth(
user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-x"
)
response = await get_router_settings(user_api_key_dict=admin_user)
assert response.current_values.get("num_retries") == 10