diff --git a/litellm/proxy/response_polling/polling_handler.py b/litellm/proxy/response_polling/polling_handler.py index 787f27f56c8..e50587b1a0e 100644 --- a/litellm/proxy/response_polling/polling_handler.py +++ b/litellm/proxy/response_polling/polling_handler.py @@ -308,10 +308,7 @@ def should_use_polling_for_request( # Otherwise, check ALL deployments for this model_name in router elif llm_router is not None: try: - # Get all deployment indices for this model name - indices: Final = llm_router.model_name_to_deployment_indices.get(model, []) - for idx in indices: - deployment_dict = llm_router.model_list[idx] + for deployment_dict in llm_router.deployments_named(model) or (): litellm_params = deployment_dict.get("litellm_params", {}) # Check custom_llm_provider first diff --git a/litellm/router.py b/litellm/router.py index 71a7f1a0463..b11b6d6b6a2 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -10620,7 +10620,7 @@ class Router: # O(k) via the name→indices map: only touch deployments whose name # is listed in `available_models`, instead of scanning model_list. for name in config.available_models: - deployments = self._deployments_named(name) + deployments = self.deployments_named(name) if not deployments: continue d = deployments[0] @@ -11477,8 +11477,8 @@ class Router: if _budget_limiter is not None: _budget_limiter.register_deployment_budget(deployment=deployment.to_json(exclude_none=True)) - def _deployments_named(self, model_name: str) -> tuple[Any, ...] | None: - """The deployments indexed under `model_name`, or None when the name is not indexed.""" + def deployments_named(self, model_name: str) -> tuple[Any, ...] | None: + """The deployments indexed under `model_name`, read under the index lock, or None when the name is not indexed.""" with self._deployment_index_lock: indices: Final = self.model_name_to_deployment_indices.get(model_name) return None if indices is None else tuple(self.model_list[idx] for idx in indices) @@ -11674,7 +11674,7 @@ class Router: this never triggers pattern matching or deep copies, so it is safe to call per listed model on the /v1/models hot path. """ - deployments: Final = self._deployments_named(model_name) + deployments: Final = self.deployments_named(model_name) if not deployments: return None @@ -12775,7 +12775,7 @@ class Router: if model_name is not None: # O(1) lookup in model_name index, then O(k) iteration where k = deployments for this model_name - for model in self._deployments_named(model_name) or (): + for model in self.deployments_named(model_name) or (): if "model_info" in model and "id" in model["model_info"]: if exclude_team_models and model["model_info"].get("team_id"): continue @@ -13129,7 +13129,7 @@ class Router: return returned_models # O(1) lookup in model_name index - named_models: Final = self._deployments_named(model_name) + named_models: Final = self.deployments_named(model_name) if named_models is not None: # O(k) where k = deployments for this model_name (typically 1-10) for model in named_models: diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index a475358e1f0..2d31d19018a 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -3126,15 +3126,13 @@ class ComplexityRouter(CustomLogger): model_to_prefs: Final[dict[str, AdaptiveRouterPreferences]] = {} model_to_cost: Final[dict[str, float]] = {} - model_list: Final = getattr(self.litellm_router_instance, "model_list", None) or [] - name_to_indices: Final = getattr(self.litellm_router_instance, "model_name_to_deployment_indices", {}) or {} for name in available_models: - indices = name_to_indices.get(name, []) - if not indices: + deployments = self.litellm_router_instance.deployments_named(name) + if not deployments: model_to_prefs[name] = AdaptiveRouterPreferences(quality_tier=2, strengths=[]) model_to_cost[name] = 0.0 continue - deployment = model_list[indices[0]] + deployment = deployments[0] mi = deployment.get("model_info") if isinstance(deployment, dict) else deployment.model_info mi_dict: dict[str, Any] = mi if isinstance(mi, dict) else (mi.model_dump() if mi else {}) prefs_raw = mi_dict.get("adaptive_router_preferences") diff --git a/litellm/router_strategy/savings_baseline.py b/litellm/router_strategy/savings_baseline.py index b5b80ddae6d..add0f78fd3a 100644 --- a/litellm/router_strategy/savings_baseline.py +++ b/litellm/router_strategy/savings_baseline.py @@ -14,7 +14,7 @@ bare string with no provider beside them; an operator who writes ``deepseek-r1`` Azure would otherwise be priced against whoever else owns that name. """ -from collections.abc import Iterable +from collections.abc import Iterable, Mapping from typing import TYPE_CHECKING, Final, NamedTuple from litellm._logging import verbose_router_logger @@ -80,12 +80,11 @@ def _models_in(router: "Router", group_name: str) -> tuple[Baseline, ...]: straight at a provider model rather than at a configured group, and prices under its own name because there is no deployment to override it. """ - indices: Final = router.model_name_to_deployment_indices.get(group_name) - if not indices: + deployments: Final = router.deployments_named(group_name) + if not deployments: return (Baseline(qualified),) if (qualified := canonical_model(group_name)) else () - def candidate(index: int) -> Baseline | None: - deployment: Final = router.model_list[index] + def candidate(deployment: Mapping[str, object]) -> Baseline | None: params: Final = deployment.get("litellm_params") if not isinstance(params, dict): return None @@ -98,7 +97,7 @@ def _models_in(router: "Router", group_name: str) -> tuple[Baseline, ...]: deployment_id: Final = info.get("id") if isinstance(info, dict) else None return Baseline(qualified, str(deployment_id) if deployment_id else None) - return tuple(c for index in indices if (c := candidate(index)) is not None) + return tuple(c for deployment in deployments if (c := candidate(deployment)) is not None) def _priced(router: "Router", candidate: Baseline) -> tuple[float, Baseline] | None: diff --git a/tests/code_coverage_tests/router_code_coverage.py b/tests/code_coverage_tests/router_code_coverage.py index 713a5a5d2ad..51773768407 100644 --- a/tests/code_coverage_tests/router_code_coverage.py +++ b/tests/code_coverage_tests/router_code_coverage.py @@ -155,7 +155,6 @@ ignored_function_names = [ "_deployment_field", # Read by is_batch_capable, tested through the mixed-group tests in test_router.py "_deployment_provider", # Read by without_deployments_that_cannot_batch, tested through the mixed-group tests in test_router.py "_cannot_run_batches", # Tested through the Azure and OpenAI mixed-group acreate_batch tests in test_router.py - "_deployments_named", # Tested through the by-name lookups in the concurrent delete tests in test_router.py "_team_deployments_named", # Tested through the team lookups in the concurrent delete tests in test_router.py ] diff --git a/tests/unit/proxy/response_polling/test_response_polling_handler.py b/tests/unit/proxy/response_polling/test_response_polling_handler.py index 979ab499f3b..63e38ce533a 100644 --- a/tests/unit/proxy/response_polling/test_response_polling_handler.py +++ b/tests/unit/proxy/response_polling/test_response_polling_handler.py @@ -1011,10 +1011,9 @@ class TestPollingConditionChecks: # Create mock router mock_router = Mock() - mock_router.model_name_to_deployment_indices = {"gpt-5": [0]} - mock_router.model_list = [ - {"model_name": "gpt-5", "litellm_params": {"model": "openai/gpt-5"}} - ] + mock_router.deployments_named.side_effect = { + "gpt-5": ({"model_name": "gpt-5", "litellm_params": {"model": "openai/gpt-5"}},) + }.get result = should_use_polling_for_request( background_mode=True, @@ -1033,13 +1032,14 @@ class TestPollingConditionChecks: ) mock_router = Mock() - mock_router.model_name_to_deployment_indices = {"claude-3": [0]} - mock_router.model_list = [ - { - "model_name": "claude-3", - "litellm_params": {"model": "anthropic/claude-3-sonnet"}, - } - ] + mock_router.deployments_named.side_effect = { + "claude-3": ( + { + "model_name": "claude-3", + "litellm_params": {"model": "anthropic/claude-3-sonnet"}, + }, + ) + }.get result = should_use_polling_for_request( background_mode=True, @@ -1183,13 +1183,14 @@ class TestPollingConditionChecks: ) mock_router = Mock() - mock_router.model_name_to_deployment_indices = {"deep-research": [0]} - mock_router.model_list = [ - { - "model_name": "deep-research", - "litellm_params": {"model": "openai/o4-mini-deep-research"}, - } - ] + mock_router.deployments_named.side_effect = { + "deep-research": ( + { + "model_name": "deep-research", + "litellm_params": {"model": "openai/o4-mini-deep-research"}, + }, + ) + }.get # Model alias "deep-research" is in native_background_mode result = should_use_polling_for_request( diff --git a/tests/unit/router_strategy/test_complexity_router.py b/tests/unit/router_strategy/test_complexity_router.py index f956bb66e6c..6d45bd69af0 100644 --- a/tests/unit/router_strategy/test_complexity_router.py +++ b/tests/unit/router_strategy/test_complexity_router.py @@ -112,6 +112,13 @@ def _heuristic_v2_artifact() -> TrainedTierArtifact: ) +def _answer_deployments_named(router: MagicMock) -> None: + """Make a mock router answer `deployments_named` from its own `model_list`, the way Router answers from its index.""" + router.deployments_named.side_effect = lambda name: ( + tuple(deployment for deployment in router.model_list if deployment["model_name"] == name) or None + ) + + @pytest.fixture def mock_router_instance(): """Create a mock LiteLLM Router instance.""" @@ -3176,7 +3183,7 @@ class TestCapabilityClassifier: {"model_name": "efficient-model", "litellm_params": {"input_cost_per_token": 0.000001}}, {"model_name": "capable-model", "litellm_params": {"input_cost_per_token": 0.00001}}, ] - mock_router_instance.model_name_to_deployment_indices = {"efficient-model": [0], "capable-model": [1]} + _answer_deployments_named(mock_router_instance) router: Final = self._router( mock_router_instance, adaptive=True, @@ -5665,7 +5672,7 @@ class TestAdaptiveSoftFloors: "model_info": {"adaptive_router_preferences": {"quality_tier": 3, "strengths": []}}, }, ] - router.model_name_to_deployment_indices = {"cheap": [0], "premium": [1]} + _answer_deployments_named(router) return router @pytest.fixture @@ -6824,7 +6831,7 @@ class TestTierModelAffinity: ) -> ComplexityRouter: mock_router_instance.cache = DualCache() mock_router_instance.model_list = [] - mock_router_instance.model_name_to_deployment_indices = {} + _answer_deployments_named(mock_router_instance) return ComplexityRouter( model_name="affinity-router", litellm_router_instance=mock_router_instance, @@ -7542,7 +7549,7 @@ class TestSessionAffinity: "model_info": {}, }, ] - mock_router_instance.model_name_to_deployment_indices = {"cheap": [0]} + _answer_deployments_named(mock_router_instance) router = ComplexityRouter( model_name="hybrid", litellm_router_instance=mock_router_instance, @@ -11346,7 +11353,7 @@ class TestClientHousekeepingCalls: "model_info": {"adaptive_router_preferences": {"quality_tier": 3, "strengths": []}}, }, ] - adaptive_instance.model_name_to_deployment_indices = {"cheap": [0], "premium": [1]} + _answer_deployments_named(adaptive_instance) router = ComplexityRouter( model_name="hybrid", litellm_router_instance=adaptive_instance, @@ -13437,7 +13444,7 @@ class TestPlanModeTierFloor: "model_info": {"adaptive_router_preferences": {"quality_tier": 3, "strengths": []}}, }, ] - adaptive_instance.model_name_to_deployment_indices = {"cheap": [0], "premium": [1]} + _answer_deployments_named(adaptive_instance) router = ComplexityRouter( model_name="hybrid", litellm_router_instance=adaptive_instance, @@ -14936,7 +14943,7 @@ class TestModalityRouting: elif path == "adaptive_pick_rewritten": config["adaptive"] = True mock_router_instance.model_list = [] - mock_router_instance.model_name_to_deployment_indices = {} + _answer_deployments_named(mock_router_instance) router = self._router(mock_router_instance, config, vision) result = await router.async_pre_routing_hook(model="m", request_kwargs=request_kwargs, messages=messages) assert result.model == expected_model diff --git a/tests/unit/router_strategy/test_savings_baseline.py b/tests/unit/router_strategy/test_savings_baseline.py index 5efc73d2dcb..4241a541202 100644 --- a/tests/unit/router_strategy/test_savings_baseline.py +++ b/tests/unit/router_strategy/test_savings_baseline.py @@ -107,8 +107,7 @@ class TestResolveBaseline: served; a dashboard counterfactual must not be able to take routing down.""" class Exploding: - @property - def model_name_to_deployment_indices(self): + def deployments_named(self, model_name): raise RuntimeError("router is mid-reload") assert resolve_baseline(Exploding(), ["anything"]) is None diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 5356af20e6b..2d2a473c6ce 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -61,7 +61,10 @@ from litellm.router import ( _without_line_breaks, Span, ) +from litellm.proxy.response_polling.polling_handler import should_use_polling_for_request from litellm.router_strategy import simple_shuffle +from litellm.router_strategy.complexity_router.complexity_router import ComplexityRouter +from litellm.router_strategy.savings_baseline import Baseline, _models_in from litellm.router_utils.client_initalization_utils import MaxParallelRequestsLimit from litellm.router_utils.cooldown_handlers import ( _async_get_cooldown_deployments, @@ -29904,6 +29907,27 @@ def _deployment_ids(deployments: list[DeploymentTypedDict]) -> tuple[str, ...]: ("deployment-2", "deployment-3"), id="team_deployments_across_teams", ), + pytest.param( + lambda router: _deployment_ids(list(router.deployments_named("model-3") or ())), + ("deployment-3",), + id="deployments_named", + ), + pytest.param( + lambda router: should_use_polling_for_request( + background_mode=True, + polling_via_cache_enabled=["openai"], + redis_cache=MagicMock(), + model="model-3", + llm_router=router, + ), + True, + id="response_polling_provider_lookup", + ), + pytest.param( + lambda router: _models_in(router, "model-3"), + (Baseline("openai/gpt-3", "deployment-3"),), + id="savings_baseline_candidates", + ), ], ) def test_lookup_during_a_concurrent_delete_returns_the_requested_deployment( @@ -29988,6 +30012,26 @@ def test_init_adaptive_router_deployment_during_a_concurrent_delete_prices_the_l assert router.adaptive_routers["adaptive"][0].strategy.model_to_cost == {"model-3": 3e-06} +def test_complexity_router_adaptive_pool_during_a_concurrent_delete_prices_the_listed_model() -> None: + priced: Final = { + **_concurrent_delete_deployment(3), + "litellm_params": {"model": "openai/gpt-3", "api_key": "k", "input_cost_per_token": 3e-06}, + } + router: Final = Router(model_list=[*(_concurrent_delete_deployment(index) for index in range(3)), priced]) + complexity: Final = ComplexityRouter( + model_name="hybrid", + litellm_router_instance=router, + complexity_router_config={"adaptive": True, "tiers": {"SIMPLE": ["model-3"]}}, + ) + + adaptive: Final = _read_during_held_write( + complexity._ensure_adaptive_router, lambda: router.delete_deployment("deployment-0") + ) + + assert adaptive is not None + assert adaptive.model_to_cost == {"model-3": 3e-06} + + def test_delete_of_an_unknown_deployment_returns_none_and_keeps_the_list() -> None: router: Final = _concurrent_delete_router()