mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(router): apply an admin pause to wildcard deployments
A request whose model is not a configured name is resolved by the pattern router in _try_early_resolve_deployments_for_model_not_in_names and returned straight from _common_checks_available_deployment, before the _filter_blocked_deployments call the exact-name path goes through. A wildcard deployment paused with model_info.blocked therefore kept serving traffic for the life of the process, while an exact-name deployment in the same proxy stopped immediately. Filter the early-resolved deployments too. When the filter removes every candidate, fall through instead of returning an empty list, so the normal path reports no healthy deployments exactly as it does for an exact-name model.
This commit is contained in:
parent
a2626726a2
commit
c41f028149
2 changed files with 81 additions and 1 deletions
|
|
@ -12498,7 +12498,14 @@ class Router:
|
|||
if early is not None:
|
||||
if not isinstance(early[1], list):
|
||||
return early
|
||||
return early[0], self._drop_strategy_markers(early[0], early[1])
|
||||
# Wildcard and team-public names resolve here and return before the
|
||||
# `_filter_blocked_deployments` call further down, so an admin pause has
|
||||
# to be applied on this path too. When it removes every candidate, fall
|
||||
# through rather than return an empty list: the normal path then reports
|
||||
# "no healthy deployments" exactly as it does for an exact-name model.
|
||||
routable_early: Final = self._filter_blocked_deployments(early[1])
|
||||
if routable_early:
|
||||
return early[0], self._drop_strategy_markers(early[0], routable_early)
|
||||
|
||||
## get healthy deployments
|
||||
### get all deployments
|
||||
|
|
|
|||
|
|
@ -16716,3 +16716,76 @@ class TestMemberAutoRouterInference:
|
|||
monkeypatch.setitem(sys.modules, "fastapi", None)
|
||||
monkeypatch.delitem(sys.modules, "litellm.proxy.auth.auto_router_checks", raising=False)
|
||||
assert (await self._route(router, {"metadata": {"user_api_key_team_id": "router-team"}})).model == "restricted-model"
|
||||
|
||||
|
||||
class TestBlockedWildcardDeployments:
|
||||
"""An admin pause has to reach wildcard deployments too.
|
||||
|
||||
A request whose model is not a configured name (`zzz-test-9` against
|
||||
`zzz-test-*`) is resolved by the pattern router and returned before the
|
||||
blocked filter the exact-name path runs, so a paused wildcard deployment kept
|
||||
serving traffic until the process restarted. See #41580.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _router(model_name: str, blocked: bool):
|
||||
import litellm
|
||||
|
||||
return litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": model_name,
|
||||
"litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-fake"},
|
||||
"model_info": {"id": "dep-wildcard", "blocked": blocked},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
def test_paused_wildcard_deployment_is_not_routable(self):
|
||||
import litellm
|
||||
|
||||
router = self._router("zzz-test-*", blocked=True)
|
||||
|
||||
with pytest.raises(litellm.BadRequestError):
|
||||
router._common_checks_available_deployment(
|
||||
model="zzz-test-9",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
request_kwargs={},
|
||||
)
|
||||
|
||||
def test_unpaused_wildcard_deployment_still_routes(self):
|
||||
router = self._router("zzz-test-*", blocked=False)
|
||||
|
||||
_, healthy = router._common_checks_available_deployment(
|
||||
model="zzz-test-9",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
request_kwargs={},
|
||||
)
|
||||
|
||||
assert [deployment["model_info"]["id"] for deployment in healthy] == ["dep-wildcard"]
|
||||
|
||||
def test_pausing_a_live_wildcard_deployment_takes_effect_without_restart(self):
|
||||
import litellm
|
||||
from litellm.types.router import Deployment, LiteLLM_Params
|
||||
|
||||
router = self._router("zzz-test-*", blocked=False)
|
||||
router._common_checks_available_deployment(
|
||||
model="zzz-test-9",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
request_kwargs={},
|
||||
)
|
||||
|
||||
router.upsert_deployment(
|
||||
deployment=Deployment(
|
||||
model_name="zzz-test-*",
|
||||
litellm_params=LiteLLM_Params(model="openai/gpt-4o", api_key="sk-fake"),
|
||||
model_info={"id": "dep-wildcard", "blocked": True},
|
||||
)
|
||||
)
|
||||
|
||||
with pytest.raises(litellm.BadRequestError):
|
||||
router._common_checks_available_deployment(
|
||||
model="zzz-test-9",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
request_kwargs={},
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue