mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
fix(router): read deployments by name under the index lock outside the Router
The response polling provider check, the complexity router's adaptive pool and the savings baseline turned the name index into list positions themselves, so a concurrent delete could still hand them a stale slot. They now read through Router.deployments_named, which takes the deployment index lock.
This commit is contained in:
parent
e1ef749651
commit
246d74e08b
9 changed files with 93 additions and 49 deletions
|
|
@ -308,10 +308,7 @@ def should_use_polling_for_request(
|
|||
# Otherwise, check ALL deployments for this model_name in router
|
||||
elif llm_router is not None:
|
||||
try:
|
||||
# Get all deployment indices for this model name
|
||||
indices: Final = llm_router.model_name_to_deployment_indices.get(model, [])
|
||||
for idx in indices:
|
||||
deployment_dict = llm_router.model_list[idx]
|
||||
for deployment_dict in llm_router.deployments_named(model) or ():
|
||||
litellm_params = deployment_dict.get("litellm_params", {})
|
||||
|
||||
# Check custom_llm_provider first
|
||||
|
|
|
|||
|
|
@ -10620,7 +10620,7 @@ class Router:
|
|||
# O(k) via the name→indices map: only touch deployments whose name
|
||||
# is listed in `available_models`, instead of scanning model_list.
|
||||
for name in config.available_models:
|
||||
deployments = self._deployments_named(name)
|
||||
deployments = self.deployments_named(name)
|
||||
if not deployments:
|
||||
continue
|
||||
d = deployments[0]
|
||||
|
|
@ -11477,8 +11477,8 @@ class Router:
|
|||
if _budget_limiter is not None:
|
||||
_budget_limiter.register_deployment_budget(deployment=deployment.to_json(exclude_none=True))
|
||||
|
||||
def _deployments_named(self, model_name: str) -> tuple[Any, ...] | None:
|
||||
"""The deployments indexed under `model_name`, or None when the name is not indexed."""
|
||||
def deployments_named(self, model_name: str) -> tuple[Any, ...] | None:
|
||||
"""The deployments indexed under `model_name`, read under the index lock, or None when the name is not indexed."""
|
||||
with self._deployment_index_lock:
|
||||
indices: Final = self.model_name_to_deployment_indices.get(model_name)
|
||||
return None if indices is None else tuple(self.model_list[idx] for idx in indices)
|
||||
|
|
@ -11674,7 +11674,7 @@ class Router:
|
|||
this never triggers pattern matching or deep copies, so it is safe to call per
|
||||
listed model on the /v1/models hot path.
|
||||
"""
|
||||
deployments: Final = self._deployments_named(model_name)
|
||||
deployments: Final = self.deployments_named(model_name)
|
||||
if not deployments:
|
||||
return None
|
||||
|
||||
|
|
@ -12775,7 +12775,7 @@ class Router:
|
|||
|
||||
if model_name is not None:
|
||||
# O(1) lookup in model_name index, then O(k) iteration where k = deployments for this model_name
|
||||
for model in self._deployments_named(model_name) or ():
|
||||
for model in self.deployments_named(model_name) or ():
|
||||
if "model_info" in model and "id" in model["model_info"]:
|
||||
if exclude_team_models and model["model_info"].get("team_id"):
|
||||
continue
|
||||
|
|
@ -13129,7 +13129,7 @@ class Router:
|
|||
return returned_models
|
||||
|
||||
# O(1) lookup in model_name index
|
||||
named_models: Final = self._deployments_named(model_name)
|
||||
named_models: Final = self.deployments_named(model_name)
|
||||
if named_models is not None:
|
||||
# O(k) where k = deployments for this model_name (typically 1-10)
|
||||
for model in named_models:
|
||||
|
|
|
|||
|
|
@ -3126,15 +3126,13 @@ class ComplexityRouter(CustomLogger):
|
|||
|
||||
model_to_prefs: Final[dict[str, AdaptiveRouterPreferences]] = {}
|
||||
model_to_cost: Final[dict[str, float]] = {}
|
||||
model_list: Final = getattr(self.litellm_router_instance, "model_list", None) or []
|
||||
name_to_indices: Final = getattr(self.litellm_router_instance, "model_name_to_deployment_indices", {}) or {}
|
||||
for name in available_models:
|
||||
indices = name_to_indices.get(name, [])
|
||||
if not indices:
|
||||
deployments = self.litellm_router_instance.deployments_named(name)
|
||||
if not deployments:
|
||||
model_to_prefs[name] = AdaptiveRouterPreferences(quality_tier=2, strengths=[])
|
||||
model_to_cost[name] = 0.0
|
||||
continue
|
||||
deployment = model_list[indices[0]]
|
||||
deployment = deployments[0]
|
||||
mi = deployment.get("model_info") if isinstance(deployment, dict) else deployment.model_info
|
||||
mi_dict: dict[str, Any] = mi if isinstance(mi, dict) else (mi.model_dump() if mi else {})
|
||||
prefs_raw = mi_dict.get("adaptive_router_preferences")
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ bare string with no provider beside them; an operator who writes ``deepseek-r1``
|
|||
Azure would otherwise be priced against whoever else owns that name.
|
||||
"""
|
||||
|
||||
from collections.abc import Iterable
|
||||
from collections.abc import Iterable, Mapping
|
||||
from typing import TYPE_CHECKING, Final, NamedTuple
|
||||
|
||||
from litellm._logging import verbose_router_logger
|
||||
|
|
@ -80,12 +80,11 @@ def _models_in(router: "Router", group_name: str) -> tuple[Baseline, ...]:
|
|||
straight at a provider model rather than at a configured group, and prices under
|
||||
its own name because there is no deployment to override it.
|
||||
"""
|
||||
indices: Final = router.model_name_to_deployment_indices.get(group_name)
|
||||
if not indices:
|
||||
deployments: Final = router.deployments_named(group_name)
|
||||
if not deployments:
|
||||
return (Baseline(qualified),) if (qualified := canonical_model(group_name)) else ()
|
||||
|
||||
def candidate(index: int) -> Baseline | None:
|
||||
deployment: Final = router.model_list[index]
|
||||
def candidate(deployment: Mapping[str, object]) -> Baseline | None:
|
||||
params: Final = deployment.get("litellm_params")
|
||||
if not isinstance(params, dict):
|
||||
return None
|
||||
|
|
@ -98,7 +97,7 @@ def _models_in(router: "Router", group_name: str) -> tuple[Baseline, ...]:
|
|||
deployment_id: Final = info.get("id") if isinstance(info, dict) else None
|
||||
return Baseline(qualified, str(deployment_id) if deployment_id else None)
|
||||
|
||||
return tuple(c for index in indices if (c := candidate(index)) is not None)
|
||||
return tuple(c for deployment in deployments if (c := candidate(deployment)) is not None)
|
||||
|
||||
|
||||
def _priced(router: "Router", candidate: Baseline) -> tuple[float, Baseline] | None:
|
||||
|
|
|
|||
|
|
@ -155,7 +155,6 @@ ignored_function_names = [
|
|||
"_deployment_field", # Read by is_batch_capable, tested through the mixed-group tests in test_router.py
|
||||
"_deployment_provider", # Read by without_deployments_that_cannot_batch, tested through the mixed-group tests in test_router.py
|
||||
"_cannot_run_batches", # Tested through the Azure and OpenAI mixed-group acreate_batch tests in test_router.py
|
||||
"_deployments_named", # Tested through the by-name lookups in the concurrent delete tests in test_router.py
|
||||
"_team_deployments_named", # Tested through the team lookups in the concurrent delete tests in test_router.py
|
||||
]
|
||||
|
||||
|
|
|
|||
|
|
@ -1011,10 +1011,9 @@ class TestPollingConditionChecks:
|
|||
|
||||
# Create mock router
|
||||
mock_router = Mock()
|
||||
mock_router.model_name_to_deployment_indices = {"gpt-5": [0]}
|
||||
mock_router.model_list = [
|
||||
{"model_name": "gpt-5", "litellm_params": {"model": "openai/gpt-5"}}
|
||||
]
|
||||
mock_router.deployments_named.side_effect = {
|
||||
"gpt-5": ({"model_name": "gpt-5", "litellm_params": {"model": "openai/gpt-5"}},)
|
||||
}.get
|
||||
|
||||
result = should_use_polling_for_request(
|
||||
background_mode=True,
|
||||
|
|
@ -1033,13 +1032,14 @@ class TestPollingConditionChecks:
|
|||
)
|
||||
|
||||
mock_router = Mock()
|
||||
mock_router.model_name_to_deployment_indices = {"claude-3": [0]}
|
||||
mock_router.model_list = [
|
||||
{
|
||||
"model_name": "claude-3",
|
||||
"litellm_params": {"model": "anthropic/claude-3-sonnet"},
|
||||
}
|
||||
]
|
||||
mock_router.deployments_named.side_effect = {
|
||||
"claude-3": (
|
||||
{
|
||||
"model_name": "claude-3",
|
||||
"litellm_params": {"model": "anthropic/claude-3-sonnet"},
|
||||
},
|
||||
)
|
||||
}.get
|
||||
|
||||
result = should_use_polling_for_request(
|
||||
background_mode=True,
|
||||
|
|
@ -1183,13 +1183,14 @@ class TestPollingConditionChecks:
|
|||
)
|
||||
|
||||
mock_router = Mock()
|
||||
mock_router.model_name_to_deployment_indices = {"deep-research": [0]}
|
||||
mock_router.model_list = [
|
||||
{
|
||||
"model_name": "deep-research",
|
||||
"litellm_params": {"model": "openai/o4-mini-deep-research"},
|
||||
}
|
||||
]
|
||||
mock_router.deployments_named.side_effect = {
|
||||
"deep-research": (
|
||||
{
|
||||
"model_name": "deep-research",
|
||||
"litellm_params": {"model": "openai/o4-mini-deep-research"},
|
||||
},
|
||||
)
|
||||
}.get
|
||||
|
||||
# Model alias "deep-research" is in native_background_mode
|
||||
result = should_use_polling_for_request(
|
||||
|
|
|
|||
|
|
@ -112,6 +112,13 @@ def _heuristic_v2_artifact() -> TrainedTierArtifact:
|
|||
)
|
||||
|
||||
|
||||
def _answer_deployments_named(router: MagicMock) -> None:
|
||||
"""Make a mock router answer `deployments_named` from its own `model_list`, the way Router answers from its index."""
|
||||
router.deployments_named.side_effect = lambda name: (
|
||||
tuple(deployment for deployment in router.model_list if deployment["model_name"] == name) or None
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_router_instance():
|
||||
"""Create a mock LiteLLM Router instance."""
|
||||
|
|
@ -3176,7 +3183,7 @@ class TestCapabilityClassifier:
|
|||
{"model_name": "efficient-model", "litellm_params": {"input_cost_per_token": 0.000001}},
|
||||
{"model_name": "capable-model", "litellm_params": {"input_cost_per_token": 0.00001}},
|
||||
]
|
||||
mock_router_instance.model_name_to_deployment_indices = {"efficient-model": [0], "capable-model": [1]}
|
||||
_answer_deployments_named(mock_router_instance)
|
||||
router: Final = self._router(
|
||||
mock_router_instance,
|
||||
adaptive=True,
|
||||
|
|
@ -5665,7 +5672,7 @@ class TestAdaptiveSoftFloors:
|
|||
"model_info": {"adaptive_router_preferences": {"quality_tier": 3, "strengths": []}},
|
||||
},
|
||||
]
|
||||
router.model_name_to_deployment_indices = {"cheap": [0], "premium": [1]}
|
||||
_answer_deployments_named(router)
|
||||
return router
|
||||
|
||||
@pytest.fixture
|
||||
|
|
@ -6824,7 +6831,7 @@ class TestTierModelAffinity:
|
|||
) -> ComplexityRouter:
|
||||
mock_router_instance.cache = DualCache()
|
||||
mock_router_instance.model_list = []
|
||||
mock_router_instance.model_name_to_deployment_indices = {}
|
||||
_answer_deployments_named(mock_router_instance)
|
||||
return ComplexityRouter(
|
||||
model_name="affinity-router",
|
||||
litellm_router_instance=mock_router_instance,
|
||||
|
|
@ -7542,7 +7549,7 @@ class TestSessionAffinity:
|
|||
"model_info": {},
|
||||
},
|
||||
]
|
||||
mock_router_instance.model_name_to_deployment_indices = {"cheap": [0]}
|
||||
_answer_deployments_named(mock_router_instance)
|
||||
router = ComplexityRouter(
|
||||
model_name="hybrid",
|
||||
litellm_router_instance=mock_router_instance,
|
||||
|
|
@ -11346,7 +11353,7 @@ class TestClientHousekeepingCalls:
|
|||
"model_info": {"adaptive_router_preferences": {"quality_tier": 3, "strengths": []}},
|
||||
},
|
||||
]
|
||||
adaptive_instance.model_name_to_deployment_indices = {"cheap": [0], "premium": [1]}
|
||||
_answer_deployments_named(adaptive_instance)
|
||||
router = ComplexityRouter(
|
||||
model_name="hybrid",
|
||||
litellm_router_instance=adaptive_instance,
|
||||
|
|
@ -13437,7 +13444,7 @@ class TestPlanModeTierFloor:
|
|||
"model_info": {"adaptive_router_preferences": {"quality_tier": 3, "strengths": []}},
|
||||
},
|
||||
]
|
||||
adaptive_instance.model_name_to_deployment_indices = {"cheap": [0], "premium": [1]}
|
||||
_answer_deployments_named(adaptive_instance)
|
||||
router = ComplexityRouter(
|
||||
model_name="hybrid",
|
||||
litellm_router_instance=adaptive_instance,
|
||||
|
|
@ -14936,7 +14943,7 @@ class TestModalityRouting:
|
|||
elif path == "adaptive_pick_rewritten":
|
||||
config["adaptive"] = True
|
||||
mock_router_instance.model_list = []
|
||||
mock_router_instance.model_name_to_deployment_indices = {}
|
||||
_answer_deployments_named(mock_router_instance)
|
||||
router = self._router(mock_router_instance, config, vision)
|
||||
result = await router.async_pre_routing_hook(model="m", request_kwargs=request_kwargs, messages=messages)
|
||||
assert result.model == expected_model
|
||||
|
|
|
|||
|
|
@ -107,8 +107,7 @@ class TestResolveBaseline:
|
|||
served; a dashboard counterfactual must not be able to take routing down."""
|
||||
|
||||
class Exploding:
|
||||
@property
|
||||
def model_name_to_deployment_indices(self):
|
||||
def deployments_named(self, model_name):
|
||||
raise RuntimeError("router is mid-reload")
|
||||
|
||||
assert resolve_baseline(Exploding(), ["anything"]) is None
|
||||
|
|
|
|||
|
|
@ -61,7 +61,10 @@ from litellm.router import (
|
|||
_without_line_breaks,
|
||||
Span,
|
||||
)
|
||||
from litellm.proxy.response_polling.polling_handler import should_use_polling_for_request
|
||||
from litellm.router_strategy import simple_shuffle
|
||||
from litellm.router_strategy.complexity_router.complexity_router import ComplexityRouter
|
||||
from litellm.router_strategy.savings_baseline import Baseline, _models_in
|
||||
from litellm.router_utils.client_initalization_utils import MaxParallelRequestsLimit
|
||||
from litellm.router_utils.cooldown_handlers import (
|
||||
_async_get_cooldown_deployments,
|
||||
|
|
@ -29904,6 +29907,27 @@ def _deployment_ids(deployments: list[DeploymentTypedDict]) -> tuple[str, ...]:
|
|||
("deployment-2", "deployment-3"),
|
||||
id="team_deployments_across_teams",
|
||||
),
|
||||
pytest.param(
|
||||
lambda router: _deployment_ids(list(router.deployments_named("model-3") or ())),
|
||||
("deployment-3",),
|
||||
id="deployments_named",
|
||||
),
|
||||
pytest.param(
|
||||
lambda router: should_use_polling_for_request(
|
||||
background_mode=True,
|
||||
polling_via_cache_enabled=["openai"],
|
||||
redis_cache=MagicMock(),
|
||||
model="model-3",
|
||||
llm_router=router,
|
||||
),
|
||||
True,
|
||||
id="response_polling_provider_lookup",
|
||||
),
|
||||
pytest.param(
|
||||
lambda router: _models_in(router, "model-3"),
|
||||
(Baseline("openai/gpt-3", "deployment-3"),),
|
||||
id="savings_baseline_candidates",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_lookup_during_a_concurrent_delete_returns_the_requested_deployment(
|
||||
|
|
@ -29988,6 +30012,26 @@ def test_init_adaptive_router_deployment_during_a_concurrent_delete_prices_the_l
|
|||
assert router.adaptive_routers["adaptive"][0].strategy.model_to_cost == {"model-3": 3e-06}
|
||||
|
||||
|
||||
def test_complexity_router_adaptive_pool_during_a_concurrent_delete_prices_the_listed_model() -> None:
|
||||
priced: Final = {
|
||||
**_concurrent_delete_deployment(3),
|
||||
"litellm_params": {"model": "openai/gpt-3", "api_key": "k", "input_cost_per_token": 3e-06},
|
||||
}
|
||||
router: Final = Router(model_list=[*(_concurrent_delete_deployment(index) for index in range(3)), priced])
|
||||
complexity: Final = ComplexityRouter(
|
||||
model_name="hybrid",
|
||||
litellm_router_instance=router,
|
||||
complexity_router_config={"adaptive": True, "tiers": {"SIMPLE": ["model-3"]}},
|
||||
)
|
||||
|
||||
adaptive: Final = _read_during_held_write(
|
||||
complexity._ensure_adaptive_router, lambda: router.delete_deployment("deployment-0")
|
||||
)
|
||||
|
||||
assert adaptive is not None
|
||||
assert adaptive.model_to_cost == {"model-3": 3e-06}
|
||||
|
||||
|
||||
def test_delete_of_an_unknown_deployment_returns_none_and_keeps_the_list() -> None:
|
||||
router: Final = _concurrent_delete_router()
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue