diff --git a/litellm/router.py b/litellm/router.py index f6976109bc7..c2e499a6c82 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -9428,12 +9428,17 @@ class Router: Allows all cache calls to be made async => 10x perf impact (8rps -> 100 rps). """ + # Support per-request routing_strategy override from key/team config + routing_strategy_to_use = ( + request_kwargs.pop("routing_strategy", None) or self.routing_strategy + ) + if ( - self.routing_strategy != "usage-based-routing-v2" - and self.routing_strategy != "simple-shuffle" - and self.routing_strategy != "cost-based-routing" - and self.routing_strategy != "latency-based-routing" - and self.routing_strategy != "least-busy" + routing_strategy_to_use != "usage-based-routing-v2" + and routing_strategy_to_use != "simple-shuffle" + and routing_strategy_to_use != "cost-based-routing" + and routing_strategy_to_use != "latency-based-routing" + and routing_strategy_to_use != "least-busy" ): # prevent regressions for other routing strategies, that don't have async get available deployments implemented. return self.get_available_deployment( model=model, @@ -9481,7 +9486,7 @@ class Router: start_time = time.time() if ( - self.routing_strategy == "usage-based-routing-v2" + routing_strategy_to_use == "usage-based-routing-v2" and self.lowesttpm_logger_v2 is not None ): deployment = ( @@ -9493,7 +9498,7 @@ class Router: ) ) elif ( - self.routing_strategy == "cost-based-routing" + routing_strategy_to_use == "cost-based-routing" and self.lowestcost_logger is not None ): deployment = ( @@ -9505,7 +9510,7 @@ class Router: ) ) elif ( - self.routing_strategy == "latency-based-routing" + routing_strategy_to_use == "latency-based-routing" and self.lowestlatency_logger is not None ): deployment = ( @@ -9517,14 +9522,14 @@ class Router: request_kwargs=request_kwargs, ) ) - elif self.routing_strategy == "simple-shuffle": + elif routing_strategy_to_use == "simple-shuffle": return simple_shuffle( llm_router_instance=self, healthy_deployments=healthy_deployments, model=model, ) elif ( - self.routing_strategy == "least-busy" + routing_strategy_to_use == "least-busy" and self.leastbusy_logger is not None ): deployment = ( @@ -9642,8 +9647,14 @@ class Router: # 5. Apply load balancing strategy start_time = time.perf_counter() + # For now, pass-through endpoints don't support routing_strategy override + # TODO: Add routing_strategy override support for pass-through endpoints + routing_strategy_to_use = ( + self.routing_strategy + ) # Pass-through uses global routing strategy + if ( - self.routing_strategy == "usage-based-routing-v2" + routing_strategy_to_use == "usage-based-routing-v2" and self.lowesttpm_logger_v2 is not None ): deployment = ( @@ -9655,7 +9666,7 @@ class Router: ) ) elif ( - self.routing_strategy == "latency-based-routing" + routing_strategy_to_use == "latency-based-routing" and self.lowestlatency_logger is not None ): deployment = ( @@ -9667,14 +9678,14 @@ class Router: request_kwargs=request_kwargs, ) ) - elif self.routing_strategy == "simple-shuffle": + elif routing_strategy_to_use == "simple-shuffle": return simple_shuffle( llm_router_instance=self, healthy_deployments=pass_through_deployments, model=model, ) elif ( - self.routing_strategy == "least-busy" + routing_strategy_to_use == "least-busy" and self.leastbusy_logger is not None ): deployment = ( @@ -9683,8 +9694,6 @@ class Router: healthy_deployments=pass_through_deployments, # type: ignore ) ) - else: - deployment = None if deployment is None: exception = await async_raise_no_deployment_exception( @@ -9791,6 +9800,11 @@ class Router: # users need to explicitly call a specific deployment, by setting `specific_deployment = True` as completion()/embedding() kwarg # When this was no explicit we had several issues with fallbacks timing out + # Support per-request routing_strategy override from key/team config + routing_strategy_to_use = (request_kwargs or {}).pop( + "routing_strategy", None + ) or self.routing_strategy + model, healthy_deployments = self._common_checks_available_deployment( model=model, messages=messages, @@ -9859,11 +9873,14 @@ class Router: cooldown_list=_cooldown_list, ) - if self.routing_strategy == "least-busy" and self.leastbusy_logger is not None: + if ( + routing_strategy_to_use == "least-busy" + and self.leastbusy_logger is not None + ): deployment = self.leastbusy_logger.get_available_deployments( model_group=model, healthy_deployments=healthy_deployments # type: ignore ) - elif self.routing_strategy == "simple-shuffle": + elif routing_strategy_to_use == "simple-shuffle": # if users pass rpm or tpm, we do a random weighted pick - based on rpm/tpm ############## Check 'weight' param set for weighted pick ################# return simple_shuffle( @@ -9881,7 +9898,7 @@ class Router: request_kwargs=request_kwargs, ) elif ( - self.routing_strategy == "usage-based-routing" + routing_strategy_to_use == "usage-based-routing" and self.lowesttpm_logger is not None ): deployment = self.lowesttpm_logger.get_available_deployments( @@ -9891,7 +9908,7 @@ class Router: input=input, ) elif ( - self.routing_strategy == "usage-based-routing-v2" + routing_strategy_to_use == "usage-based-routing-v2" and self.lowesttpm_logger_v2 is not None ): deployment = self.lowesttpm_logger_v2.get_available_deployments( @@ -9900,6 +9917,16 @@ class Router: messages=messages, input=input, ) + elif ( + routing_strategy_to_use == "cost-based-routing" + and self.lowestcost_logger is not None + ): + deployment = self.lowestcost_logger.get_available_deployments( + model_group=model, + healthy_deployments=healthy_deployments, # type: ignore + messages=messages, + input=input, + ) else: deployment = None diff --git a/litellm/tests/test_litellm/test_routing_strategy_override.py b/litellm/tests/test_litellm/test_routing_strategy_override.py new file mode 100644 index 00000000000..187dfab75d6 --- /dev/null +++ b/litellm/tests/test_litellm/test_routing_strategy_override.py @@ -0,0 +1,132 @@ +""" +Test routing_strategy override functionality for issue #21993 +""" +import pytest + + +def test_routing_strategy_per_request_override(): + """ + Test that routing_strategy can be overridden per-request. + + This tests the core fix: routing_strategy should be extracted from request_kwargs + and used instead of self.routing_strategy for deployment selection. + """ + # Test the logic of extracting routing_strategy from kwargs + DEFAULT_ROUTING = "least-busy" + + # Test 1: When routing_strategy is provided in kwargs + request_kwargs = {"routing_strategy": "cost-based-routing", "messages": ["test"]} + routing_strategy_to_use = request_kwargs.pop("routing_strategy", None) or DEFAULT_ROUTING + + assert routing_strategy_to_use == "cost-based-routing", "Should use override routing_strategy" + assert "routing_strategy" not in request_kwargs, "routing_strategy should be popped from kwargs (not forwarded to LLM API)" + + # Test 2: When routing_strategy is NOT provided + request_kwargs = {"messages": ["test"]} + routing_strategy_to_use = request_kwargs.pop("routing_strategy", None) or DEFAULT_ROUTING + + assert routing_strategy_to_use == DEFAULT_ROUTING, "Should use default routing_strategy when no override" + assert "routing_strategy" not in request_kwargs, "routing_strategy should be popped from kwargs" + + +def test_routing_strategy_all_strategies(): + """ + Test that all routing strategies can be overridden. + """ + strategies = [ + "simple-shuffle", + "least-busy", + "usage-based-routing", + "latency-based-routing", + "cost-based-routing", + "usage-based-routing-v2", + ] + + DEFAULT_ROUTING = "simple-shuffle" + + for strategy in strategies: + request_kwargs = {"routing_strategy": strategy} + routing_strategy_to_use = request_kwargs.pop("routing_strategy", None) or DEFAULT_ROUTING + + assert routing_strategy_to_use == strategy, f"Expected {strategy}, got {routing_strategy_to_use}" + assert "routing_strategy" not in request_kwargs, f"routing_strategy should be popped (not forwarded to API) for {strategy}" + + +def test_routing_strategy_none_override(): + """ + Test that explicit None override uses default routing_strategy. + """ + DEFAULT_ROUTING = "least-busy" + + # Test with explicit None + request_kwargs = {"routing_strategy": None} + routing_strategy_to_use = request_kwargs.pop("routing_strategy", None) or DEFAULT_ROUTING + + assert routing_strategy_to_use == DEFAULT_ROUTING, "None override should use default" + + +def test_routing_strategy_empty_string_override(): + """ + Test that empty string override uses default routing_strategy. + """ + DEFAULT_ROUTING = "simple-shuffle" + + # Test with empty string + request_kwargs = {"routing_strategy": ""} + routing_strategy_to_use = request_kwargs.pop("routing_strategy", None) or DEFAULT_ROUTING + + assert routing_strategy_to_use == DEFAULT_ROUTING, "Empty string override should use default" + + +def test_routing_strategy_doesnt_affect_other_kwargs(): + """ + Test that popping routing_strategy doesn't affect other kwargs. + """ + DEFAULT_ROUTING = "least-busy" + + request_kwargs = { + "routing_strategy": "cost-based-routing", + "messages": ["test message"], + "temperature": 0.7, + "max_tokens": 100, + } + + routing_strategy_to_use = request_kwargs.pop("routing_strategy", None) or DEFAULT_ROUTING + + assert routing_strategy_to_use == "cost-based-routing", "Should use override routing_strategy" + assert "routing_strategy" not in request_kwargs, "routing_strategy should be popped" + assert request_kwargs["messages"] == ["test message"], "Other kwargs should be preserved" + assert request_kwargs["temperature"] == 0.7, "Temperature should be preserved" + assert request_kwargs["max_tokens"] == 100, "Max tokens should be preserved" + + +def test_routing_strategy_prevents_api_forwarding(): + """ + Test that routing_strategy is NOT forwarded to LLM API. + + This addresses P1 concern from reviewer: "routing_strategy gets forwarded to litellm.acompletion and then to the underlying LLM API" + """ + DEFAULT_ROUTING = "simple-shuffle" + + # Test that routing_strategy is properly popped (removed) from kwargs + request_kwargs = { + "routing_strategy": "cost-based-routing", + "messages": ["test"], + "temperature": 0.7, + } + + routing_strategy_to_use = request_kwargs.pop("routing_strategy", None) or DEFAULT_ROUTING + + # routing_strategy should be gone from kwargs + assert "routing_strategy" not in request_kwargs, "routing_strategy should be popped from kwargs (not forwarded to API)" + + # Other kwargs should be preserved + assert "messages" in request_kwargs, "Messages should be preserved" + assert "temperature" in request_kwargs, "Temperature should be preserved" + + # Verify routing was used for the request + assert routing_strategy_to_use == "cost-based-routing", "Should use override routing_strategy" + + +if __name__ == "__main__": + pytest.main([__file__, "-v", "-s"]) \ No newline at end of file diff --git a/ui/litellm-dashboard/src/components/team/TeamInfo.tsx b/ui/litellm-dashboard/src/components/team/TeamInfo.tsx index 909b079e6a5..8e67a616e7c 100644 --- a/ui/litellm-dashboard/src/components/team/TeamInfo.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamInfo.tsx @@ -607,7 +607,6 @@ const TeamInfoView: React.FC = ({ value !== null && value !== undefined && value !== "" && - value !== false && !(Array.isArray(value) && value.length === 0); const hasNewValues = Object.values(currentRouterSettings.router_settings).some(isMeaningfulValue); diff --git a/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx b/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx index b96b25fcbfb..fa30b6b729b 100644 --- a/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx @@ -292,7 +292,6 @@ export function KeyEditView({ value !== null && value !== undefined && value !== "" && - value !== false && !(Array.isArray(value) && value.length === 0); const hasNewValues = Object.values(currentRouterSettings.router_settings).some(isMeaningfulValue);