mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(proxy): keep live model groups routable when a sibling deployment is dropped
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
1f98f908c5
commit
bd33f686bb
4 changed files with 40 additions and 45 deletions
|
|
@ -6511,7 +6511,7 @@ class ProxyConfig:
|
|||
|
||||
added_models = 0
|
||||
## ADD MODEL LOGIC
|
||||
for m in sorted(db_models, key=lambda m: (getattr(m, "created_at", None) or datetime.min, m.model_id)):
|
||||
for m in sorted(db_models, key=lambda m: (m.created_at or datetime.min, m.model_id)):
|
||||
_litellm_params = m.litellm_params
|
||||
if isinstance(_litellm_params, dict):
|
||||
# decrypt values
|
||||
|
|
@ -15051,9 +15051,9 @@ def _model_info_json_response(
|
|||
data: Sequence[Mapping[str, object]] | Mapping[str, object],
|
||||
dropped_deployments: Sequence[Mapping[str, object]] | None = None,
|
||||
) -> Response:
|
||||
response_data: Final = {"data": data}
|
||||
if dropped_deployments is not None:
|
||||
response_data["dropped_deployments"] = dropped_deployments
|
||||
response_data: Final = (
|
||||
{"data": data} if dropped_deployments is None else {"data": data, "dropped_deployments": dropped_deployments}
|
||||
)
|
||||
return Response(
|
||||
content=orjson.dumps(response_data, default=jsonable_encoder, option=orjson.OPT_NON_STR_KEYS),
|
||||
media_type="application/json",
|
||||
|
|
@ -15276,7 +15276,7 @@ async def model_info_v1(
|
|||
verbose_proxy_logger.debug("all_models: %s", visible_models)
|
||||
dropped_deployments: Final = (
|
||||
[asdict(dropped_deployment) for dropped_deployment in llm_router.dropped_deployments.values()]
|
||||
if hasattr(user_api_key_dict, "user_role") and _user_has_admin_view(user_api_key_dict)
|
||||
if _user_has_admin_view(user_api_key_dict)
|
||||
else None
|
||||
)
|
||||
return _model_info_json_response(visible_models, dropped_deployments=dropped_deployments)
|
||||
|
|
|
|||
|
|
@ -491,14 +491,7 @@ async def _route_request_single_attempt( # noqa: ANN202 # returns unawaited pr
|
|||
data.pop("enable_tag_filtering", None)
|
||||
|
||||
team_id: Final = get_team_id_from_data(data)
|
||||
dropped_model_names: Final = (
|
||||
frozenset(dropped_deployment.model_name for dropped_deployment in llm_router.dropped_deployments.values())
|
||||
if llm_router is not None
|
||||
else frozenset()
|
||||
)
|
||||
router_model_names: Final = (
|
||||
(frozenset(llm_router.model_names) - dropped_model_names) if llm_router is not None else frozenset()
|
||||
)
|
||||
router_model_names: Final = llm_router.model_names if llm_router is not None else []
|
||||
is_proxy_admin_without_team: Final = team_id is None and _is_proxy_admin_request(data)
|
||||
|
||||
# Preprocess Google GenAI generate content requests
|
||||
|
|
@ -656,14 +649,11 @@ async def _route_request_single_attempt( # noqa: ANN202 # returns unawaited pr
|
|||
data["model"] = team_model_name
|
||||
return getattr(llm_router, f"{route_type}")(**data)
|
||||
|
||||
elif data["model"] not in dropped_model_names and (
|
||||
(
|
||||
is_proxy_admin_without_team
|
||||
and data["model"] not in router_model_names
|
||||
and data["model"] in llm_router.team_public_model_names
|
||||
)
|
||||
or llm_router.is_recognized_model(data["model"])
|
||||
):
|
||||
elif (
|
||||
is_proxy_admin_without_team
|
||||
and data["model"] not in router_model_names
|
||||
and data["model"] in llm_router.team_public_model_names
|
||||
) or llm_router.is_recognized_model(data["model"]):
|
||||
return getattr(llm_router, f"{route_type}")(**data)
|
||||
|
||||
elif data["model"] not in router_model_names:
|
||||
|
|
|
|||
|
|
@ -5206,11 +5206,12 @@ async def test_model_info_v1_oci_secrets_not_leaked():
|
|||
from litellm.proxy.proxy_server import model_info_v1
|
||||
|
||||
# Mock user authentication
|
||||
mock_user_api_key_dict = MagicMock(spec=UserAPIKeyAuth)
|
||||
mock_user_api_key_dict.user_id = "test-user"
|
||||
mock_user_api_key_dict.api_key = "test-key"
|
||||
mock_user_api_key_dict.team_models = []
|
||||
mock_user_api_key_dict.models = ["oci-grok-test"]
|
||||
mock_user_api_key_dict = UserAPIKeyAuth(
|
||||
user_id="test-user",
|
||||
api_key="test-key",
|
||||
team_models=[],
|
||||
models=["oci-grok-test"],
|
||||
)
|
||||
|
||||
# Mock model data with OCI sensitive information
|
||||
mock_model_data = {
|
||||
|
|
|
|||
|
|
@ -1,11 +1,10 @@
|
|||
|
||||
import pytest
|
||||
|
||||
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Final
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from fastapi import HTTPException
|
||||
|
||||
from litellm.proxy.route_llm_request import ProxyModelNotFoundError, route_request
|
||||
|
|
@ -54,7 +53,7 @@ async def test_route_request_distinguishes_dropped_deployments_from_unknown_mode
|
|||
"litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "fake"},
|
||||
},
|
||||
{
|
||||
"model_name": "x",
|
||||
"model_name": "live",
|
||||
"litellm_params": {
|
||||
"model": "auto_router/complexity_router",
|
||||
"complexity_router_config": {
|
||||
|
|
@ -62,28 +61,32 @@ async def test_route_request_distinguishes_dropped_deployments_from_unknown_mode
|
|||
"tiers": {"SIMPLE": "gpt-4o-mini", "MEDIUM": "gpt-4o"},
|
||||
},
|
||||
},
|
||||
"model_info": {"id": "id-x"},
|
||||
},
|
||||
{
|
||||
"model_name": "y",
|
||||
"litellm_params": {
|
||||
"model": "auto_router/complexity_router",
|
||||
"complexity_router_config": {
|
||||
"classifier_type": "heuristic_v2",
|
||||
"tiers": {"SIMPLE": "gpt-4o-mini", "MEDIUM": "gpt-4o"},
|
||||
},
|
||||
},
|
||||
"model_info": {"id": "id-y"},
|
||||
"model_info": {"id": "id-live"},
|
||||
},
|
||||
],
|
||||
auto_router_capability_limit=lambda: 1,
|
||||
ignore_invalid_deployments=True,
|
||||
)
|
||||
assert "id-y" in router.dropped_deployments
|
||||
from litellm.types.router import Deployment
|
||||
|
||||
assert router.upsert_deployment(
|
||||
Deployment(
|
||||
model_name="x",
|
||||
litellm_params={
|
||||
"model": "auto_router/complexity_router",
|
||||
"complexity_router_config": {
|
||||
"classifier_type": "heuristic_v2",
|
||||
"tiers": {"SIMPLE": "gpt-4o-mini", "MEDIUM": "gpt-4o"},
|
||||
},
|
||||
},
|
||||
model_info={"id": "id-x"},
|
||||
)
|
||||
) is None
|
||||
assert "id-x" in router.dropped_deployments
|
||||
|
||||
with pytest.raises(HTTPException) as dropped_error:
|
||||
await route_request(
|
||||
data={"model": "y", "messages": [{"role": "user", "content": "Hello"}]},
|
||||
data={"model": "x", "messages": [{"role": "user", "content": "Hello"}]},
|
||||
llm_router=router,
|
||||
user_model=None,
|
||||
route_type="acompletion",
|
||||
|
|
@ -1205,6 +1208,7 @@ def _db_model_row(model_name: str, mock_response: str):
|
|||
litellm_params={"model": "openai/gpt-4o", "api_key": "fake", "mock_response": mock_response},
|
||||
model_info={},
|
||||
blocked=False,
|
||||
created_at=datetime.min,
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue