fix(router): read deployments by name under the index lock outside the Router

The response polling provider check, the complexity router's adaptive pool and the
savings baseline turned the name index into list positions themselves, so a concurrent
delete could still hand them a stale slot. They now read through Router.deployments_named,
which takes the deployment index lock.
This commit is contained in:
yucheng-berri 2026-10-26 12:42:57 -07:00
parent e1ef749651
commit 246d74e08b
9 changed files with 93 additions and 49 deletions

View file

@ -308,10 +308,7 @@ def should_use_polling_for_request(
# Otherwise, check ALL deployments for this model_name in router
elif llm_router is not None:
try:
# Get all deployment indices for this model name
indices: Final = llm_router.model_name_to_deployment_indices.get(model, [])
for idx in indices:
deployment_dict = llm_router.model_list[idx]
for deployment_dict in llm_router.deployments_named(model) or ():
litellm_params = deployment_dict.get("litellm_params", {})
# Check custom_llm_provider first

View file

@ -10620,7 +10620,7 @@ class Router:
# O(k) via the name→indices map: only touch deployments whose name
# is listed in `available_models`, instead of scanning model_list.
for name in config.available_models:
deployments = self._deployments_named(name)
deployments = self.deployments_named(name)
if not deployments:
continue
d = deployments[0]
@ -11477,8 +11477,8 @@ class Router:
if _budget_limiter is not None:
_budget_limiter.register_deployment_budget(deployment=deployment.to_json(exclude_none=True))
def _deployments_named(self, model_name: str) -> tuple[Any, ...] | None:
"""The deployments indexed under `model_name`, or None when the name is not indexed."""
def deployments_named(self, model_name: str) -> tuple[Any, ...] | None:
"""The deployments indexed under `model_name`, read under the index lock, or None when the name is not indexed."""
with self._deployment_index_lock:
indices: Final = self.model_name_to_deployment_indices.get(model_name)
return None if indices is None else tuple(self.model_list[idx] for idx in indices)
@ -11674,7 +11674,7 @@ class Router:
this never triggers pattern matching or deep copies, so it is safe to call per
listed model on the /v1/models hot path.
"""
deployments: Final = self._deployments_named(model_name)
deployments: Final = self.deployments_named(model_name)
if not deployments:
return None
@ -12775,7 +12775,7 @@ class Router:
if model_name is not None:
# O(1) lookup in model_name index, then O(k) iteration where k = deployments for this model_name
for model in self._deployments_named(model_name) or ():
for model in self.deployments_named(model_name) or ():
if "model_info" in model and "id" in model["model_info"]:
if exclude_team_models and model["model_info"].get("team_id"):
continue
@ -13129,7 +13129,7 @@ class Router:
return returned_models
# O(1) lookup in model_name index
named_models: Final = self._deployments_named(model_name)
named_models: Final = self.deployments_named(model_name)
if named_models is not None:
# O(k) where k = deployments for this model_name (typically 1-10)
for model in named_models:

View file

@ -3126,15 +3126,13 @@ class ComplexityRouter(CustomLogger):
model_to_prefs: Final[dict[str, AdaptiveRouterPreferences]] = {}
model_to_cost: Final[dict[str, float]] = {}
model_list: Final = getattr(self.litellm_router_instance, "model_list", None) or []
name_to_indices: Final = getattr(self.litellm_router_instance, "model_name_to_deployment_indices", {}) or {}
for name in available_models:
indices = name_to_indices.get(name, [])
if not indices:
deployments = self.litellm_router_instance.deployments_named(name)
if not deployments:
model_to_prefs[name] = AdaptiveRouterPreferences(quality_tier=2, strengths=[])
model_to_cost[name] = 0.0
continue
deployment = model_list[indices[0]]
deployment = deployments[0]
mi = deployment.get("model_info") if isinstance(deployment, dict) else deployment.model_info
mi_dict: dict[str, Any] = mi if isinstance(mi, dict) else (mi.model_dump() if mi else {})
prefs_raw = mi_dict.get("adaptive_router_preferences")

View file

@ -14,7 +14,7 @@ bare string with no provider beside them; an operator who writes ``deepseek-r1``
Azure would otherwise be priced against whoever else owns that name.
"""
from collections.abc import Iterable
from collections.abc import Iterable, Mapping
from typing import TYPE_CHECKING, Final, NamedTuple
from litellm._logging import verbose_router_logger
@ -80,12 +80,11 @@ def _models_in(router: "Router", group_name: str) -> tuple[Baseline, ...]:
straight at a provider model rather than at a configured group, and prices under
its own name because there is no deployment to override it.
"""
indices: Final = router.model_name_to_deployment_indices.get(group_name)
if not indices:
deployments: Final = router.deployments_named(group_name)
if not deployments:
return (Baseline(qualified),) if (qualified := canonical_model(group_name)) else ()
def candidate(index: int) -> Baseline | None:
deployment: Final = router.model_list[index]
def candidate(deployment: Mapping[str, object]) -> Baseline | None:
params: Final = deployment.get("litellm_params")
if not isinstance(params, dict):
return None
@ -98,7 +97,7 @@ def _models_in(router: "Router", group_name: str) -> tuple[Baseline, ...]:
deployment_id: Final = info.get("id") if isinstance(info, dict) else None
return Baseline(qualified, str(deployment_id) if deployment_id else None)
return tuple(c for index in indices if (c := candidate(index)) is not None)
return tuple(c for deployment in deployments if (c := candidate(deployment)) is not None)
def _priced(router: "Router", candidate: Baseline) -> tuple[float, Baseline] | None:

View file

@ -155,7 +155,6 @@ ignored_function_names = [
"_deployment_field", # Read by is_batch_capable, tested through the mixed-group tests in test_router.py
"_deployment_provider", # Read by without_deployments_that_cannot_batch, tested through the mixed-group tests in test_router.py
"_cannot_run_batches", # Tested through the Azure and OpenAI mixed-group acreate_batch tests in test_router.py
"_deployments_named", # Tested through the by-name lookups in the concurrent delete tests in test_router.py
"_team_deployments_named", # Tested through the team lookups in the concurrent delete tests in test_router.py
]

View file

@ -1011,10 +1011,9 @@ class TestPollingConditionChecks:
# Create mock router
mock_router = Mock()
mock_router.model_name_to_deployment_indices = {"gpt-5": [0]}
mock_router.model_list = [
{"model_name": "gpt-5", "litellm_params": {"model": "openai/gpt-5"}}
]
mock_router.deployments_named.side_effect = {
"gpt-5": ({"model_name": "gpt-5", "litellm_params": {"model": "openai/gpt-5"}},)
}.get
result = should_use_polling_for_request(
background_mode=True,
@ -1033,13 +1032,14 @@ class TestPollingConditionChecks:
)
mock_router = Mock()
mock_router.model_name_to_deployment_indices = {"claude-3": [0]}
mock_router.model_list = [
{
"model_name": "claude-3",
"litellm_params": {"model": "anthropic/claude-3-sonnet"},
}
]
mock_router.deployments_named.side_effect = {
"claude-3": (
{
"model_name": "claude-3",
"litellm_params": {"model": "anthropic/claude-3-sonnet"},
},
)
}.get
result = should_use_polling_for_request(
background_mode=True,
@ -1183,13 +1183,14 @@ class TestPollingConditionChecks:
)
mock_router = Mock()
mock_router.model_name_to_deployment_indices = {"deep-research": [0]}
mock_router.model_list = [
{
"model_name": "deep-research",
"litellm_params": {"model": "openai/o4-mini-deep-research"},
}
]
mock_router.deployments_named.side_effect = {
"deep-research": (
{
"model_name": "deep-research",
"litellm_params": {"model": "openai/o4-mini-deep-research"},
},
)
}.get
# Model alias "deep-research" is in native_background_mode
result = should_use_polling_for_request(

View file

@ -112,6 +112,13 @@ def _heuristic_v2_artifact() -> TrainedTierArtifact:
)
def _answer_deployments_named(router: MagicMock) -> None:
"""Make a mock router answer `deployments_named` from its own `model_list`, the way Router answers from its index."""
router.deployments_named.side_effect = lambda name: (
tuple(deployment for deployment in router.model_list if deployment["model_name"] == name) or None
)
@pytest.fixture
def mock_router_instance():
"""Create a mock LiteLLM Router instance."""
@ -3176,7 +3183,7 @@ class TestCapabilityClassifier:
{"model_name": "efficient-model", "litellm_params": {"input_cost_per_token": 0.000001}},
{"model_name": "capable-model", "litellm_params": {"input_cost_per_token": 0.00001}},
]
mock_router_instance.model_name_to_deployment_indices = {"efficient-model": [0], "capable-model": [1]}
_answer_deployments_named(mock_router_instance)
router: Final = self._router(
mock_router_instance,
adaptive=True,
@ -5665,7 +5672,7 @@ class TestAdaptiveSoftFloors:
"model_info": {"adaptive_router_preferences": {"quality_tier": 3, "strengths": []}},
},
]
router.model_name_to_deployment_indices = {"cheap": [0], "premium": [1]}
_answer_deployments_named(router)
return router
@pytest.fixture
@ -6824,7 +6831,7 @@ class TestTierModelAffinity:
) -> ComplexityRouter:
mock_router_instance.cache = DualCache()
mock_router_instance.model_list = []
mock_router_instance.model_name_to_deployment_indices = {}
_answer_deployments_named(mock_router_instance)
return ComplexityRouter(
model_name="affinity-router",
litellm_router_instance=mock_router_instance,
@ -7542,7 +7549,7 @@ class TestSessionAffinity:
"model_info": {},
},
]
mock_router_instance.model_name_to_deployment_indices = {"cheap": [0]}
_answer_deployments_named(mock_router_instance)
router = ComplexityRouter(
model_name="hybrid",
litellm_router_instance=mock_router_instance,
@ -11346,7 +11353,7 @@ class TestClientHousekeepingCalls:
"model_info": {"adaptive_router_preferences": {"quality_tier": 3, "strengths": []}},
},
]
adaptive_instance.model_name_to_deployment_indices = {"cheap": [0], "premium": [1]}
_answer_deployments_named(adaptive_instance)
router = ComplexityRouter(
model_name="hybrid",
litellm_router_instance=adaptive_instance,
@ -13437,7 +13444,7 @@ class TestPlanModeTierFloor:
"model_info": {"adaptive_router_preferences": {"quality_tier": 3, "strengths": []}},
},
]
adaptive_instance.model_name_to_deployment_indices = {"cheap": [0], "premium": [1]}
_answer_deployments_named(adaptive_instance)
router = ComplexityRouter(
model_name="hybrid",
litellm_router_instance=adaptive_instance,
@ -14936,7 +14943,7 @@ class TestModalityRouting:
elif path == "adaptive_pick_rewritten":
config["adaptive"] = True
mock_router_instance.model_list = []
mock_router_instance.model_name_to_deployment_indices = {}
_answer_deployments_named(mock_router_instance)
router = self._router(mock_router_instance, config, vision)
result = await router.async_pre_routing_hook(model="m", request_kwargs=request_kwargs, messages=messages)
assert result.model == expected_model

View file

@ -107,8 +107,7 @@ class TestResolveBaseline:
served; a dashboard counterfactual must not be able to take routing down."""
class Exploding:
@property
def model_name_to_deployment_indices(self):
def deployments_named(self, model_name):
raise RuntimeError("router is mid-reload")
assert resolve_baseline(Exploding(), ["anything"]) is None

View file

@ -61,7 +61,10 @@ from litellm.router import (
_without_line_breaks,
Span,
)
from litellm.proxy.response_polling.polling_handler import should_use_polling_for_request
from litellm.router_strategy import simple_shuffle
from litellm.router_strategy.complexity_router.complexity_router import ComplexityRouter
from litellm.router_strategy.savings_baseline import Baseline, _models_in
from litellm.router_utils.client_initalization_utils import MaxParallelRequestsLimit
from litellm.router_utils.cooldown_handlers import (
_async_get_cooldown_deployments,
@ -29904,6 +29907,27 @@ def _deployment_ids(deployments: list[DeploymentTypedDict]) -> tuple[str, ...]:
("deployment-2", "deployment-3"),
id="team_deployments_across_teams",
),
pytest.param(
lambda router: _deployment_ids(list(router.deployments_named("model-3") or ())),
("deployment-3",),
id="deployments_named",
),
pytest.param(
lambda router: should_use_polling_for_request(
background_mode=True,
polling_via_cache_enabled=["openai"],
redis_cache=MagicMock(),
model="model-3",
llm_router=router,
),
True,
id="response_polling_provider_lookup",
),
pytest.param(
lambda router: _models_in(router, "model-3"),
(Baseline("openai/gpt-3", "deployment-3"),),
id="savings_baseline_candidates",
),
],
)
def test_lookup_during_a_concurrent_delete_returns_the_requested_deployment(
@ -29988,6 +30012,26 @@ def test_init_adaptive_router_deployment_during_a_concurrent_delete_prices_the_l
assert router.adaptive_routers["adaptive"][0].strategy.model_to_cost == {"model-3": 3e-06}
def test_complexity_router_adaptive_pool_during_a_concurrent_delete_prices_the_listed_model() -> None:
priced: Final = {
**_concurrent_delete_deployment(3),
"litellm_params": {"model": "openai/gpt-3", "api_key": "k", "input_cost_per_token": 3e-06},
}
router: Final = Router(model_list=[*(_concurrent_delete_deployment(index) for index in range(3)), priced])
complexity: Final = ComplexityRouter(
model_name="hybrid",
litellm_router_instance=router,
complexity_router_config={"adaptive": True, "tiers": {"SIMPLE": ["model-3"]}},
)
adaptive: Final = _read_during_held_write(
complexity._ensure_adaptive_router, lambda: router.delete_deployment("deployment-0")
)
assert adaptive is not None
assert adaptive.model_to_cost == {"model-3": 3e-06}
def test_delete_of_an_unknown_deployment_returns_none_and_keeps_the_list() -> None:
router: Final = _concurrent_delete_router()