From 246d74e08b190d4b8dab05d69acfe66f94470709 Mon Sep 17 00:00:00 2001 From: yucheng-berri <293926326+yucheng-berri@users.noreply.github.com> Date: Mon, 26 Oct 2026 12:42:57 -0700 Subject: [PATCH] fix(router): read deployments by name under the index lock outside the Router The response polling provider check, the complexity router's adaptive pool and the savings baseline turned the name index into list positions themselves, so a concurrent delete could still hand them a stale slot. They now read through Router.deployments_named, which takes the deployment index lock. --- .../proxy/response_polling/polling_handler.py | 5 +-- litellm/router.py | 12 ++--- .../complexity_router/complexity_router.py | 8 ++-- litellm/router_strategy/savings_baseline.py | 11 +++-- .../router_code_coverage.py | 1 - .../test_response_polling_handler.py | 37 ++++++++-------- .../router_strategy/test_complexity_router.py | 21 ++++++--- .../router_strategy/test_savings_baseline.py | 3 +- tests/unit/test_router/test_router.py | 44 +++++++++++++++++++ 9 files changed, 93 insertions(+), 49 deletions(-) diff --git a/litellm/proxy/response_polling/polling_handler.py b/litellm/proxy/response_polling/polling_handler.py index 787f27f56c8..e50587b1a0e 100644 --- a/litellm/proxy/response_polling/polling_handler.py +++ b/litellm/proxy/response_polling/polling_handler.py @@ -308,10 +308,7 @@ def should_use_polling_for_request( # Otherwise, check ALL deployments for this model_name in router elif llm_router is not None: try: - # Get all deployment indices for this model name - indices: Final = llm_router.model_name_to_deployment_indices.get(model, []) - for idx in indices: - deployment_dict = llm_router.model_list[idx] + for deployment_dict in llm_router.deployments_named(model) or (): litellm_params = deployment_dict.get("litellm_params", {}) # Check custom_llm_provider first diff --git a/litellm/router.py b/litellm/router.py index 71a7f1a0463..b11b6d6b6a2 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -10620,7 +10620,7 @@ class Router: # O(k) via the name→indices map: only touch deployments whose name # is listed in `available_models`, instead of scanning model_list. for name in config.available_models: - deployments = self._deployments_named(name) + deployments = self.deployments_named(name) if not deployments: continue d = deployments[0] @@ -11477,8 +11477,8 @@ class Router: if _budget_limiter is not None: _budget_limiter.register_deployment_budget(deployment=deployment.to_json(exclude_none=True)) - def _deployments_named(self, model_name: str) -> tuple[Any, ...] | None: - """The deployments indexed under `model_name`, or None when the name is not indexed.""" + def deployments_named(self, model_name: str) -> tuple[Any, ...] | None: + """The deployments indexed under `model_name`, read under the index lock, or None when the name is not indexed.""" with self._deployment_index_lock: indices: Final = self.model_name_to_deployment_indices.get(model_name) return None if indices is None else tuple(self.model_list[idx] for idx in indices) @@ -11674,7 +11674,7 @@ class Router: this never triggers pattern matching or deep copies, so it is safe to call per listed model on the /v1/models hot path. """ - deployments: Final = self._deployments_named(model_name) + deployments: Final = self.deployments_named(model_name) if not deployments: return None @@ -12775,7 +12775,7 @@ class Router: if model_name is not None: # O(1) lookup in model_name index, then O(k) iteration where k = deployments for this model_name - for model in self._deployments_named(model_name) or (): + for model in self.deployments_named(model_name) or (): if "model_info" in model and "id" in model["model_info"]: if exclude_team_models and model["model_info"].get("team_id"): continue @@ -13129,7 +13129,7 @@ class Router: return returned_models # O(1) lookup in model_name index - named_models: Final = self._deployments_named(model_name) + named_models: Final = self.deployments_named(model_name) if named_models is not None: # O(k) where k = deployments for this model_name (typically 1-10) for model in named_models: diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index a475358e1f0..2d31d19018a 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -3126,15 +3126,13 @@ class ComplexityRouter(CustomLogger): model_to_prefs: Final[dict[str, AdaptiveRouterPreferences]] = {} model_to_cost: Final[dict[str, float]] = {} - model_list: Final = getattr(self.litellm_router_instance, "model_list", None) or [] - name_to_indices: Final = getattr(self.litellm_router_instance, "model_name_to_deployment_indices", {}) or {} for name in available_models: - indices = name_to_indices.get(name, []) - if not indices: + deployments = self.litellm_router_instance.deployments_named(name) + if not deployments: model_to_prefs[name] = AdaptiveRouterPreferences(quality_tier=2, strengths=[]) model_to_cost[name] = 0.0 continue - deployment = model_list[indices[0]] + deployment = deployments[0] mi = deployment.get("model_info") if isinstance(deployment, dict) else deployment.model_info mi_dict: dict[str, Any] = mi if isinstance(mi, dict) else (mi.model_dump() if mi else {}) prefs_raw = mi_dict.get("adaptive_router_preferences") diff --git a/litellm/router_strategy/savings_baseline.py b/litellm/router_strategy/savings_baseline.py index b5b80ddae6d..add0f78fd3a 100644 --- a/litellm/router_strategy/savings_baseline.py +++ b/litellm/router_strategy/savings_baseline.py @@ -14,7 +14,7 @@ bare string with no provider beside them; an operator who writes ``deepseek-r1`` Azure would otherwise be priced against whoever else owns that name. """ -from collections.abc import Iterable +from collections.abc import Iterable, Mapping from typing import TYPE_CHECKING, Final, NamedTuple from litellm._logging import verbose_router_logger @@ -80,12 +80,11 @@ def _models_in(router: "Router", group_name: str) -> tuple[Baseline, ...]: straight at a provider model rather than at a configured group, and prices under its own name because there is no deployment to override it. """ - indices: Final = router.model_name_to_deployment_indices.get(group_name) - if not indices: + deployments: Final = router.deployments_named(group_name) + if not deployments: return (Baseline(qualified),) if (qualified := canonical_model(group_name)) else () - def candidate(index: int) -> Baseline | None: - deployment: Final = router.model_list[index] + def candidate(deployment: Mapping[str, object]) -> Baseline | None: params: Final = deployment.get("litellm_params") if not isinstance(params, dict): return None @@ -98,7 +97,7 @@ def _models_in(router: "Router", group_name: str) -> tuple[Baseline, ...]: deployment_id: Final = info.get("id") if isinstance(info, dict) else None return Baseline(qualified, str(deployment_id) if deployment_id else None) - return tuple(c for index in indices if (c := candidate(index)) is not None) + return tuple(c for deployment in deployments if (c := candidate(deployment)) is not None) def _priced(router: "Router", candidate: Baseline) -> tuple[float, Baseline] | None: diff --git a/tests/code_coverage_tests/router_code_coverage.py b/tests/code_coverage_tests/router_code_coverage.py index 713a5a5d2ad..51773768407 100644 --- a/tests/code_coverage_tests/router_code_coverage.py +++ b/tests/code_coverage_tests/router_code_coverage.py @@ -155,7 +155,6 @@ ignored_function_names = [ "_deployment_field", # Read by is_batch_capable, tested through the mixed-group tests in test_router.py "_deployment_provider", # Read by without_deployments_that_cannot_batch, tested through the mixed-group tests in test_router.py "_cannot_run_batches", # Tested through the Azure and OpenAI mixed-group acreate_batch tests in test_router.py - "_deployments_named", # Tested through the by-name lookups in the concurrent delete tests in test_router.py "_team_deployments_named", # Tested through the team lookups in the concurrent delete tests in test_router.py ] diff --git a/tests/unit/proxy/response_polling/test_response_polling_handler.py b/tests/unit/proxy/response_polling/test_response_polling_handler.py index 979ab499f3b..63e38ce533a 100644 --- a/tests/unit/proxy/response_polling/test_response_polling_handler.py +++ b/tests/unit/proxy/response_polling/test_response_polling_handler.py @@ -1011,10 +1011,9 @@ class TestPollingConditionChecks: # Create mock router mock_router = Mock() - mock_router.model_name_to_deployment_indices = {"gpt-5": [0]} - mock_router.model_list = [ - {"model_name": "gpt-5", "litellm_params": {"model": "openai/gpt-5"}} - ] + mock_router.deployments_named.side_effect = { + "gpt-5": ({"model_name": "gpt-5", "litellm_params": {"model": "openai/gpt-5"}},) + }.get result = should_use_polling_for_request( background_mode=True, @@ -1033,13 +1032,14 @@ class TestPollingConditionChecks: ) mock_router = Mock() - mock_router.model_name_to_deployment_indices = {"claude-3": [0]} - mock_router.model_list = [ - { - "model_name": "claude-3", - "litellm_params": {"model": "anthropic/claude-3-sonnet"}, - } - ] + mock_router.deployments_named.side_effect = { + "claude-3": ( + { + "model_name": "claude-3", + "litellm_params": {"model": "anthropic/claude-3-sonnet"}, + }, + ) + }.get result = should_use_polling_for_request( background_mode=True, @@ -1183,13 +1183,14 @@ class TestPollingConditionChecks: ) mock_router = Mock() - mock_router.model_name_to_deployment_indices = {"deep-research": [0]} - mock_router.model_list = [ - { - "model_name": "deep-research", - "litellm_params": {"model": "openai/o4-mini-deep-research"}, - } - ] + mock_router.deployments_named.side_effect = { + "deep-research": ( + { + "model_name": "deep-research", + "litellm_params": {"model": "openai/o4-mini-deep-research"}, + }, + ) + }.get # Model alias "deep-research" is in native_background_mode result = should_use_polling_for_request( diff --git a/tests/unit/router_strategy/test_complexity_router.py b/tests/unit/router_strategy/test_complexity_router.py index f956bb66e6c..6d45bd69af0 100644 --- a/tests/unit/router_strategy/test_complexity_router.py +++ b/tests/unit/router_strategy/test_complexity_router.py @@ -112,6 +112,13 @@ def _heuristic_v2_artifact() -> TrainedTierArtifact: ) +def _answer_deployments_named(router: MagicMock) -> None: + """Make a mock router answer `deployments_named` from its own `model_list`, the way Router answers from its index.""" + router.deployments_named.side_effect = lambda name: ( + tuple(deployment for deployment in router.model_list if deployment["model_name"] == name) or None + ) + + @pytest.fixture def mock_router_instance(): """Create a mock LiteLLM Router instance.""" @@ -3176,7 +3183,7 @@ class TestCapabilityClassifier: {"model_name": "efficient-model", "litellm_params": {"input_cost_per_token": 0.000001}}, {"model_name": "capable-model", "litellm_params": {"input_cost_per_token": 0.00001}}, ] - mock_router_instance.model_name_to_deployment_indices = {"efficient-model": [0], "capable-model": [1]} + _answer_deployments_named(mock_router_instance) router: Final = self._router( mock_router_instance, adaptive=True, @@ -5665,7 +5672,7 @@ class TestAdaptiveSoftFloors: "model_info": {"adaptive_router_preferences": {"quality_tier": 3, "strengths": []}}, }, ] - router.model_name_to_deployment_indices = {"cheap": [0], "premium": [1]} + _answer_deployments_named(router) return router @pytest.fixture @@ -6824,7 +6831,7 @@ class TestTierModelAffinity: ) -> ComplexityRouter: mock_router_instance.cache = DualCache() mock_router_instance.model_list = [] - mock_router_instance.model_name_to_deployment_indices = {} + _answer_deployments_named(mock_router_instance) return ComplexityRouter( model_name="affinity-router", litellm_router_instance=mock_router_instance, @@ -7542,7 +7549,7 @@ class TestSessionAffinity: "model_info": {}, }, ] - mock_router_instance.model_name_to_deployment_indices = {"cheap": [0]} + _answer_deployments_named(mock_router_instance) router = ComplexityRouter( model_name="hybrid", litellm_router_instance=mock_router_instance, @@ -11346,7 +11353,7 @@ class TestClientHousekeepingCalls: "model_info": {"adaptive_router_preferences": {"quality_tier": 3, "strengths": []}}, }, ] - adaptive_instance.model_name_to_deployment_indices = {"cheap": [0], "premium": [1]} + _answer_deployments_named(adaptive_instance) router = ComplexityRouter( model_name="hybrid", litellm_router_instance=adaptive_instance, @@ -13437,7 +13444,7 @@ class TestPlanModeTierFloor: "model_info": {"adaptive_router_preferences": {"quality_tier": 3, "strengths": []}}, }, ] - adaptive_instance.model_name_to_deployment_indices = {"cheap": [0], "premium": [1]} + _answer_deployments_named(adaptive_instance) router = ComplexityRouter( model_name="hybrid", litellm_router_instance=adaptive_instance, @@ -14936,7 +14943,7 @@ class TestModalityRouting: elif path == "adaptive_pick_rewritten": config["adaptive"] = True mock_router_instance.model_list = [] - mock_router_instance.model_name_to_deployment_indices = {} + _answer_deployments_named(mock_router_instance) router = self._router(mock_router_instance, config, vision) result = await router.async_pre_routing_hook(model="m", request_kwargs=request_kwargs, messages=messages) assert result.model == expected_model diff --git a/tests/unit/router_strategy/test_savings_baseline.py b/tests/unit/router_strategy/test_savings_baseline.py index 5efc73d2dcb..4241a541202 100644 --- a/tests/unit/router_strategy/test_savings_baseline.py +++ b/tests/unit/router_strategy/test_savings_baseline.py @@ -107,8 +107,7 @@ class TestResolveBaseline: served; a dashboard counterfactual must not be able to take routing down.""" class Exploding: - @property - def model_name_to_deployment_indices(self): + def deployments_named(self, model_name): raise RuntimeError("router is mid-reload") assert resolve_baseline(Exploding(), ["anything"]) is None diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 5356af20e6b..2d2a473c6ce 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -61,7 +61,10 @@ from litellm.router import ( _without_line_breaks, Span, ) +from litellm.proxy.response_polling.polling_handler import should_use_polling_for_request from litellm.router_strategy import simple_shuffle +from litellm.router_strategy.complexity_router.complexity_router import ComplexityRouter +from litellm.router_strategy.savings_baseline import Baseline, _models_in from litellm.router_utils.client_initalization_utils import MaxParallelRequestsLimit from litellm.router_utils.cooldown_handlers import ( _async_get_cooldown_deployments, @@ -29904,6 +29907,27 @@ def _deployment_ids(deployments: list[DeploymentTypedDict]) -> tuple[str, ...]: ("deployment-2", "deployment-3"), id="team_deployments_across_teams", ), + pytest.param( + lambda router: _deployment_ids(list(router.deployments_named("model-3") or ())), + ("deployment-3",), + id="deployments_named", + ), + pytest.param( + lambda router: should_use_polling_for_request( + background_mode=True, + polling_via_cache_enabled=["openai"], + redis_cache=MagicMock(), + model="model-3", + llm_router=router, + ), + True, + id="response_polling_provider_lookup", + ), + pytest.param( + lambda router: _models_in(router, "model-3"), + (Baseline("openai/gpt-3", "deployment-3"),), + id="savings_baseline_candidates", + ), ], ) def test_lookup_during_a_concurrent_delete_returns_the_requested_deployment( @@ -29988,6 +30012,26 @@ def test_init_adaptive_router_deployment_during_a_concurrent_delete_prices_the_l assert router.adaptive_routers["adaptive"][0].strategy.model_to_cost == {"model-3": 3e-06} +def test_complexity_router_adaptive_pool_during_a_concurrent_delete_prices_the_listed_model() -> None: + priced: Final = { + **_concurrent_delete_deployment(3), + "litellm_params": {"model": "openai/gpt-3", "api_key": "k", "input_cost_per_token": 3e-06}, + } + router: Final = Router(model_list=[*(_concurrent_delete_deployment(index) for index in range(3)), priced]) + complexity: Final = ComplexityRouter( + model_name="hybrid", + litellm_router_instance=router, + complexity_router_config={"adaptive": True, "tiers": {"SIMPLE": ["model-3"]}}, + ) + + adaptive: Final = _read_during_held_write( + complexity._ensure_adaptive_router, lambda: router.delete_deployment("deployment-0") + ) + + assert adaptive is not None + assert adaptive.model_to_cost == {"model-3": 3e-06} + + def test_delete_of_an_unknown_deployment_returns_none_and_keeps_the_list() -> None: router: Final = _concurrent_delete_router()