mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-08 03:08:45 +00:00
litellm/proxy: preserve model order of /v1/models and /model_group/info (#13178)
Closes #12644 Signed-off-by: Alexander Yastrebov <alexander.yastrebov@zalando.de>
This commit is contained in:
Alexander Yastrebov
GitHub
parent
c427531c9c
commit
825923e7be
4 changed files with 107 additions and 24 deletions
|
|
@ -175,22 +175,27 @@ def get_complete_model_list(
|
|||
If list contains wildcard -> return known provider models
|
||||
"""
|
||||
|
||||
unique_models: Set[str] = set()
|
||||
unique_models = []
|
||||
def append_unique(models):
|
||||
for model in models:
|
||||
if model not in unique_models:
|
||||
unique_models.append(model)
|
||||
|
||||
if key_models:
|
||||
unique_models.update(key_models)
|
||||
append_unique(key_models)
|
||||
elif team_models:
|
||||
unique_models.update(team_models)
|
||||
append_unique(team_models)
|
||||
else:
|
||||
unique_models.update(proxy_model_list)
|
||||
append_unique(proxy_model_list)
|
||||
if include_model_access_groups:
|
||||
unique_models.update(model_access_groups.keys())
|
||||
append_unique(list(model_access_groups.keys())) # TODO: keys order
|
||||
|
||||
if user_model:
|
||||
unique_models.add(user_model)
|
||||
append_unique([user_model])
|
||||
|
||||
if infer_model_from_keys:
|
||||
valid_models = get_valid_models()
|
||||
unique_models.update(valid_models)
|
||||
append_unique(valid_models)
|
||||
|
||||
if only_model_access_groups:
|
||||
model_access_groups_to_return: List[str] = []
|
||||
|
|
@ -205,7 +210,7 @@ def get_complete_model_list(
|
|||
llm_router=llm_router,
|
||||
)
|
||||
|
||||
complete_model_list = list(unique_models) + all_wildcard_models
|
||||
complete_model_list = unique_models + all_wildcard_models
|
||||
|
||||
return complete_model_list
|
||||
|
||||
|
|
@ -261,7 +266,7 @@ def get_known_models_from_wildcard(
|
|||
|
||||
|
||||
def _get_wildcard_models(
|
||||
unique_models: Set[str],
|
||||
unique_models: List[str],
|
||||
return_wildcard_routes: Optional[bool] = False,
|
||||
llm_router: Optional[Router] = None,
|
||||
) -> List[str]:
|
||||
|
|
|
|||
|
|
@ -6469,18 +6469,18 @@ async def model_metrics_exceptions(
|
|||
"""
|
||||
sql_query = """
|
||||
WITH cte AS (
|
||||
SELECT
|
||||
SELECT
|
||||
CASE WHEN api_base = '' THEN litellm_model_name ELSE CONCAT(litellm_model_name, '-', api_base) END AS combined_model_api_base,
|
||||
exception_type,
|
||||
COUNT(*) AS num_rate_limit_exceptions
|
||||
FROM "LiteLLM_ErrorLogs"
|
||||
WHERE
|
||||
"startTime" >= $1::timestamp
|
||||
AND "endTime" <= $2::timestamp
|
||||
WHERE
|
||||
"startTime" >= $1::timestamp
|
||||
AND "endTime" <= $2::timestamp
|
||||
AND model_group = $3
|
||||
GROUP BY combined_model_api_base, exception_type
|
||||
)
|
||||
SELECT
|
||||
SELECT
|
||||
combined_model_api_base,
|
||||
COUNT(*) AS total_exceptions,
|
||||
json_object_agg(exception_type, num_rate_limit_exceptions) AS exception_counts
|
||||
|
|
@ -6711,10 +6711,13 @@ def _get_model_group_info(
|
|||
llm_router: Router, all_models_str: List[str], model_group: Optional[str]
|
||||
) -> List[ModelGroupInfoProxy]:
|
||||
model_groups: List[ModelGroupInfoProxy] = []
|
||||
# ensure all_models_str is a set
|
||||
all_models_str_set = set(all_models_str)
|
||||
|
||||
for model in all_models_str_set:
|
||||
unique_models = []
|
||||
for model in all_models_str:
|
||||
if model not in unique_models:
|
||||
unique_models.append(model)
|
||||
|
||||
for model in unique_models:
|
||||
if model_group is not None and model_group != model:
|
||||
continue
|
||||
|
||||
|
|
@ -7304,7 +7307,7 @@ async def login(request: Request): # noqa: PLR0915
|
|||
get_disabled_non_admin_personal_key_creation()
|
||||
)
|
||||
"""
|
||||
To login to Admin UI, we support the following
|
||||
To login to Admin UI, we support the following
|
||||
- Login with UI_USERNAME and UI_PASSWORD
|
||||
- Login with Invite Link `user_email` and `password` combination
|
||||
"""
|
||||
|
|
@ -8131,8 +8134,8 @@ async def update_config_general_settings(
|
|||
"""
|
||||
- Check if prisma_client is None
|
||||
- Check if user allowed to call this endpoint (admin-only)
|
||||
- Check if param in general settings
|
||||
- Check if config value is valid type
|
||||
- Check if param in general settings
|
||||
- Check if config value is valid type
|
||||
"""
|
||||
|
||||
if prisma_client is None:
|
||||
|
|
@ -8208,7 +8211,7 @@ async def get_config_general_settings(
|
|||
"""
|
||||
- Check if prisma_client is None
|
||||
- Check if user allowed to call this endpoint (admin-only)
|
||||
- Check if param in general settings
|
||||
- Check if param in general settings
|
||||
"""
|
||||
if prisma_client is None:
|
||||
raise HTTPException(
|
||||
|
|
@ -8272,7 +8275,7 @@ async def get_config_list(
|
|||
"""
|
||||
- Check if prisma_client is None
|
||||
- Check if user allowed to call this endpoint (admin-only)
|
||||
- Check if param in general settings
|
||||
- Check if param in general settings
|
||||
"""
|
||||
if prisma_client is None:
|
||||
raise HTTPException(
|
||||
|
|
@ -8408,7 +8411,7 @@ async def delete_config_general_settings(
|
|||
"""
|
||||
- Check if prisma_client is None
|
||||
- Check if user allowed to call this endpoint (admin-only)
|
||||
- Check if param in general settings
|
||||
- Check if param in general settings
|
||||
"""
|
||||
if prisma_client is None:
|
||||
raise HTTPException(
|
||||
|
|
@ -8592,7 +8595,7 @@ async def get_config(): # noqa: PLR0915
|
|||
},
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
"""
|
||||
for _callback in _success_callbacks:
|
||||
if _callback != "langfuse":
|
||||
|
|
|
|||
|
|
@ -19,3 +19,46 @@ def test_get_team_models_for_all_models_and_team_only_models():
|
|||
)
|
||||
combined_models = team_models + proxy_model_list
|
||||
assert set(result) == set(combined_models)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"key_models,team_models,proxy_model_list,model_list,expected",
|
||||
[
|
||||
(
|
||||
["anthropic/claude-3-haiku-20240307", "anthropic/claude-3-5-haiku-20241022"],
|
||||
[],
|
||||
[],
|
||||
[{"model_name": "anthropic/*", "litellm_params": {"model": "anthropic/*"}}],
|
||||
["anthropic/claude-3-haiku-20240307", "anthropic/claude-3-5-haiku-20241022"]
|
||||
),
|
||||
(
|
||||
[],
|
||||
["anthropic/claude-3-haiku-20240307", "anthropic/claude-3-5-haiku-20241022"],
|
||||
[],
|
||||
[{"model_name": "anthropic/*", "litellm_params": {"model": "anthropic/*"}}],
|
||||
["anthropic/claude-3-haiku-20240307", "anthropic/claude-3-5-haiku-20241022"]
|
||||
),
|
||||
(
|
||||
[],
|
||||
[],
|
||||
["anthropic/claude-3-haiku-20240307", "anthropic/claude-3-5-haiku-20241022"],
|
||||
[{"model_name": "anthropic/*", "litellm_params": {"model": "anthropic/*"}}],
|
||||
["anthropic/claude-3-haiku-20240307", "anthropic/claude-3-5-haiku-20241022"]
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_get_complete_model_list_order(key_models, team_models, proxy_model_list, model_list, expected):
|
||||
"""
|
||||
Test that get_complete_model_list preserves order
|
||||
"""
|
||||
from litellm.proxy.auth.model_checks import get_complete_model_list
|
||||
from litellm import Router
|
||||
|
||||
assert get_complete_model_list(
|
||||
proxy_model_list=proxy_model_list,
|
||||
key_models=key_models,
|
||||
team_models=team_models,
|
||||
user_model=None,
|
||||
infer_model_from_keys=False,
|
||||
llm_router=Router(model_list=model_list),
|
||||
) == expected
|
||||
|
|
|
|||
|
|
@ -57,3 +57,35 @@ def test_proxy_only_error_false_for_other_error_type():
|
|||
)
|
||||
is False
|
||||
)
|
||||
|
||||
|
||||
def test_get_model_group_info_order():
|
||||
from litellm.proxy.proxy_server import _get_model_group_info
|
||||
from litellm import Router
|
||||
|
||||
router = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "openai/tts-1",
|
||||
"litellm_params": {
|
||||
"model": "openai/tts-1",
|
||||
"api_key": "sk-1234",
|
||||
},
|
||||
},
|
||||
{
|
||||
"model_name": "openai/gpt-3.5-turbo",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-3.5-turbo",
|
||||
"api_key": "sk-1234",
|
||||
},
|
||||
},
|
||||
]
|
||||
)
|
||||
model_list = _get_model_group_info(
|
||||
llm_router=router,
|
||||
all_models_str=["openai/tts-1", "openai/gpt-3.5-turbo"],
|
||||
model_group=None,
|
||||
)
|
||||
|
||||
model_groups = [m.model_group for m in model_list]
|
||||
assert model_groups == ["openai/tts-1", "openai/gpt-3.5-turbo"]
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue