From 3b9c9e0523c3376dcad020d0603405a854b4a715 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Tue, 22 Sep 2026 13:09:38 -0700 Subject: [PATCH] feat(router): add group-scoped priority routing strategy (#42378) * feat(router): add group-scoped priority routing strategy * fix(router): satisfy priority routing type-discipline checks --- .../router_settings_endpoints.py | 13 +- litellm/router.py | 79 ++- .../add_retry_fallback_headers.py | 13 + .../router_utils/fallback_event_handlers.py | 7 +- .../deployment_affinity_check.py | 7 +- .../prompt_caching_deployment_check.py | 7 +- litellm/router_utils/routing_groups.py | 27 +- .../router_settings_endpoints.py | 1 + litellm/types/router.py | 21 +- .../test_router_settings_endpoints.py | 96 +++- .../test_router_routing_groups.py | 477 +++++++++++++++++- .../hooks/router/useRouterFields.ts | 1 + .../routingGroups/useRoutingGroups.test.ts | 65 +++ .../hooks/routingGroups/useRoutingGroups.ts | 3 +- ...=> RoutingGroupModal.integration.test.tsx} | 143 +++++- .../routing_groups/RoutingGroupModal.tsx | 120 ++++- .../routing_groups/RoutingGroupUsagePanel.tsx | 16 +- ...> RoutingGroupsTable.integration.test.tsx} | 16 + .../src/components/routing_groups/index.tsx | 14 +- .../routing_groups/modelOwnership.test.ts | 17 +- .../routing_groups/modelOwnership.ts | 6 +- .../routingGroupPayload.test.ts | 137 ++++- .../routing_groups/routingGroupPayload.ts | 62 ++- .../src/components/routing_groups/strategy.ts | 1 + .../src/components/routing_groups/types.ts | 8 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 17 + 26 files changed, 1309 insertions(+), 65 deletions(-) create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/routingGroups/useRoutingGroups.test.ts rename ui/litellm-dashboard/src/components/routing_groups/{RoutingGroupModal.test.tsx => RoutingGroupModal.integration.test.tsx} (67%) rename ui/litellm-dashboard/src/components/routing_groups/{RoutingGroupsTable.test.tsx => RoutingGroupsTable.integration.test.tsx} (86%) diff --git a/litellm/proxy/management_endpoints/router_settings_endpoints.py b/litellm/proxy/management_endpoints/router_settings_endpoints.py index d6d74ada35a..a557e3a6082 100644 --- a/litellm/proxy/management_endpoints/router_settings_endpoints.py +++ b/litellm/proxy/management_endpoints/router_settings_endpoints.py @@ -33,6 +33,9 @@ class RouterSettingsResponse(BaseModel): fields: list[RouterSettingsField] = Field(description="List of all configurable router settings with metadata") current_values: dict[str, Any] = Field(description="Current values of router settings") routing_strategy_descriptions: dict[str, str] = Field(description="Descriptions for each routing strategy option") + routing_group_strategies: tuple[str, ...] = Field( + description="Strategies supported when constructing a routing group" + ) source: dict[str, FieldSource] = Field(description="Source of each current router setting") @@ -41,6 +44,9 @@ class RouterFieldsResponse(BaseModel): description="List of all configurable router settings with metadata (without field values)" ) routing_strategy_descriptions: dict[str, str] = Field(description="Descriptions for each routing strategy option") + routing_group_strategies: tuple[str, ...] = Field( + description="Strategies supported when constructing a routing group" + ) def _router_setting_source( @@ -114,7 +120,10 @@ async def get_router_settings( if llm_router is not None: # Router exposes routing groups as private `_routing_groups`; the # generic `hasattr` loop below would miss them. - current_values["routing_groups"] = [group.model_dump() for group in llm_router._routing_groups.values()] + current_values["routing_groups"] = [ + group.model_dump(exclude=frozenset(("model_priorities",)) if group.model_priorities is None else None) + for group in llm_router._routing_groups.values() + ] for field in router_fields: if field.field_name == "routing_groups": continue @@ -147,6 +156,7 @@ async def get_router_settings( fields=router_fields, current_values=current_values, routing_strategy_descriptions=ROUTING_STRATEGY_DESCRIPTIONS, + routing_group_strategies=(*available_routing_strategies, "priority"), source=source, ) except Exception as e: @@ -196,6 +206,7 @@ async def get_router_fields( return RouterFieldsResponse( fields=router_fields, routing_strategy_descriptions=ROUTING_STRATEGY_DESCRIPTIONS, + routing_group_strategies=(*available_routing_strategies, "priority"), ) except Exception as e: verbose_proxy_logger.error("Error fetching router fields: %s", e) diff --git a/litellm/router.py b/litellm/router.py index 54b4d04a0cb..7267f6eb3ba 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -146,6 +146,7 @@ from litellm.router_utils.add_retry_fallback_headers import ( add_retry_headers_to_response, apply_quality_router_decision_headers, apply_remaining_usage_headers, + apply_response_model_id, complexity_router_decision_headers, ensure_response_additional_headers, get_hidden_params_dict, @@ -205,6 +206,7 @@ from litellm.router_utils.fallback_event_handlers import ( fallback_lookup_groups, fallbacks_disabled_for_request, get_fallback_model_group_for_lookup_groups, + get_pre_routing_selection, has_unattempted_fallback_target, mid_stream_fallback_hop_kwargs, per_request_fallback_controls, @@ -247,7 +249,11 @@ from litellm.router_utils.router_callbacks.track_deployment_metrics import ( increment_deployment_failures_for_current_minute, increment_deployment_successes_for_current_minute, ) -from litellm.router_utils.routing_groups import parse_routing_groups, validate_routing_strategy +from litellm.router_utils.routing_groups import ( + apply_routing_group_priority, + parse_routing_groups, + validate_routing_strategy, +) from litellm.scheduler import FlowItem, Scheduler from litellm.types.llms.openai import ( AllMessageValues, @@ -841,7 +847,9 @@ class Router: cooldown_time (float): Time to cooldown a deployment after failure in seconds. Defaults to 1. routing_strategy (Literal["simple-shuffle", "least-busy", "usage-based-routing", "latency-based-routing", "cost-based-routing"]): Routing strategy used for the implicit "default" group (any model not claimed by an entry in `routing_groups`). Defaults to "simple-shuffle". routing_strategy_args (dict): Additional args for the default group's routing strategy (e.g. latency window). Defaults to {}. - routing_groups (Optional[List[RoutingGroup]]): Named subsets of `model_name`s that use a per-group routing strategy and args. Each model belongs to at most one explicit group; everything else lands in the implicit "default" group driven by `routing_strategy` / `routing_strategy_args`. Defaults to None. + routing_groups (Optional[List[RoutingGroup]]): Named subsets of `model_name`s with a group routing strategy. + Priority groups apply only to group calls and may overlap. Other groups supply their members' default + strategy, with at most one such group per model. Unclaimed models use the top-level strategy. alerting_config (AlertingConfig): Slack alerting configuration. Defaults to None. provider_budget_config (ProviderBudgetConfig): Provider budget configuration. Use this to set llm_provider budget limits. example $100/day to OpenAI, $100/day to Azure, etc. Defaults to None. deployment_affinity_ttl_seconds (int): TTL for user-key -> deployment affinity mapping. Defaults to 3600. @@ -1449,10 +1457,10 @@ class Router: ) -> None: """ Validates and indexes `routing_groups`. Each `model_name` may belong to - at most one explicit group. Constructs per-group strategy selectors so + at most one non-priority group. Constructs per-group strategy selectors so groups with different `routing_strategy_args` track independent state. - Models not claimed by any explicit group are served by the implicit + Models not claimed by a non-priority group are served by the implicit `"default"` group, whose selectors are the `self._logger` attributes set up in `routing_strategy_init`. """ @@ -1466,6 +1474,8 @@ class Router: alias_names: Final = frozenset(self.model_group_alias or ()) for group in groups: if group.group_name in known_model_names or group.group_name in alias_names: + if group.routing_strategy == "priority": + raise ValueError("Priority routing group names must not shadow a model or alias") verbose_router_logger.warning( "routing_groups: group_name '%s' is shadowed by an existing model_name or model_group_alias; " "the group's strategy still applies to its members, but the name is not callable until renamed.", @@ -1476,7 +1486,7 @@ class Router: ( group, self._build_strategy_selector( - strategy=group.routing_strategy, + strategy="simple-shuffle" if group.routing_strategy == "priority" else group.routing_strategy, routing_strategy_args=group.routing_strategy_args or {}, register_callbacks=False, ), @@ -1501,7 +1511,10 @@ class Router: self._routing_groups: dict[str, RoutingGroup] = {group.group_name: group for group, _ in built} self._model_to_group: dict[str, str] = { - model_name: group.group_name for group, _ in built for model_name in group.models + model_name: group.group_name + for group, _ in built + if group.routing_strategy != "priority" + for model_name in group.models } self._group_selectors: dict[str, dict[str, RouterStrategySelector]] = { group.group_name: ( @@ -1551,11 +1564,16 @@ class Router: if routing_group is None: return None return [ # mutable-ok: matches _get_all_deployments' list contract expected by downstream filters - deployment + apply_routing_group_priority(routing_group, member, deployment) for member in routing_group.models for deployment in self._get_all_deployments(model_name=member, team_id=team_id) ] + def _is_priority_routing_group(self, model: str) -> bool: + resolved: Final = self._get_model_from_alias(model=model) or model + group: Final = self.get_routing_group(resolved) + return group is not None and group.routing_strategy == "priority" + def is_recognized_model(self, model: str) -> bool: """ Whether `model` names something this router serves directly: a @@ -1719,7 +1737,12 @@ class Router: self._bind_override_selector_to_request(override, override_selector, request_kwargs) return override, override_selector - group_name: Final = model if self.get_routing_group(model) is not None else self._model_to_group.get(model) + resolved_model: Final = self._get_model_from_alias(model=model) or model + group_name: Final = ( + resolved_model + if self.get_routing_group(resolved_model) is not None + else self._model_to_group.get(resolved_model) + ) if group_name is None: strategy = self._normalize_strategy(self.routing_strategy) attr: Final = self._DEFAULT_SELECTOR_ATTR_BY_STRATEGY.get(strategy or "") @@ -1728,6 +1751,8 @@ class Router: return strategy, selector group: Final = self._routing_groups[group_name] + if group.routing_strategy == "priority": + return "simple-shuffle", None strategy = self._normalize_strategy(group.routing_strategy) selector = self._group_selectors.get(group_name, {}).get(strategy or "") verbose_router_logger.debug("routing_group=%s model=%s strategy=%s", group_name, model, strategy) @@ -2244,6 +2269,7 @@ class Router: enable_responses_api_affinity=False, enable_session_id_affinity=False, model_group_affinity_config=self.model_group_affinity_config, + is_priority_group=self._is_priority_routing_group, ) self.optional_callbacks.append(affinity_callback) litellm.logging_callback_manager.add_litellm_callback(affinity_callback) @@ -2289,6 +2315,7 @@ class Router: enable_responses_api_affinity=enable_responses_api_affinity, enable_session_id_affinity=enable_session_id_affinity, model_group_affinity_config=self.model_group_affinity_config, + is_priority_group=self._is_priority_routing_group, ) self.optional_callbacks.append(affinity_callback) litellm.logging_callback_manager.add_litellm_callback(affinity_callback) @@ -2313,7 +2340,9 @@ class Router: ): continue if pre_call_check == "prompt_caching": - _callback = PromptCachingDeploymentCheck(cache=self.cache) + _callback = PromptCachingDeploymentCheck( + cache=self.cache, is_priority_group=self._is_priority_routing_group + ) elif pre_call_check == "router_budget_limiting": if self._get_router_deployment_budget_limiter() is not None: continue @@ -7067,7 +7096,7 @@ class Router: self, exception: Exception, original_model_group: str, - all_deployments: list[DeploymentTypedDict], + all_deployments: Sequence[DeploymentTypedDict], args: tuple, kwargs: dict, input_kwargs: dict, @@ -7193,7 +7222,8 @@ class Router: _request_team_id: Final[str | None] = (kwargs.get("metadata", {}) or {}).get("user_api_key_team_id") # Use wildcard-aware lookup so order-based fallback also works for model # groups resolved via pattern routing (e.g. `openai/*` -> `openai/gpt-4.1-mini`). - all_deployments: Final = self.get_model_list(model_name=original_model_group, team_id=_request_team_id) or [] + order_model_group: Final = get_pre_routing_selection(kwargs) or original_model_group + all_deployments: Final = self.get_model_list(model_name=order_model_group, team_id=_request_team_id) or () _order_set: Final[set] = { litellm.utils._get_deployment_order(d) for d in all_deployments @@ -7206,7 +7236,7 @@ class Router: skip_up_to: Final = current_target if current_target is not None else order_values[0] # Build order-based fallback entries (skip already-tried levels) order_fallback_entries: Final[list] = [ - {"model": original_model_group, "_target_order": o} for o in order_values if o > skip_up_to + {"model": order_model_group, "_target_order": o} for o in order_values if o > skip_up_to ] # Get external fallbacks — handle both standard and non-standard formats external_fallback_group: list | None = None @@ -11192,6 +11222,9 @@ class Router: return response additional_headers: Final = ensure_response_additional_headers(response) + apply_response_model_id( + response, find_deployment_metadata(request_kwargs) if request_kwargs is not None else None + ) additional_headers["x-litellm-model-group"] = model_group apply_quality_router_decision_headers(additional_headers, request_kwargs) additional_headers.update(complexity_router_decision_headers(request_kwargs)) @@ -11682,7 +11715,14 @@ class Router: else: continue - returned_models.extend(self._get_all_deployments(model_name=_router_model_name, model_alias=model_alias)) + if (alias_group := self.get_routing_group(_router_model_name)) is not None: + returned_models.extend( + {**row, "model_name": model_alias} for row in self._materialize_routing_group_rows((alias_group,)) + ) + else: + returned_models.extend( + self._get_all_deployments(model_name=_router_model_name, model_alias=model_alias) + ) return returned_models @@ -11713,7 +11753,7 @@ class Router: def _materialize_routing_group_rows(self, groups: tuple[RoutingGroup, ...]) -> tuple[DeploymentTypedDict, ...]: return tuple( - self._as_routing_group_row(deployment) + self._as_routing_group_row(apply_routing_group_priority(group, member, deployment)) for group in groups for member in group.models for deployment in self._get_all_deployments(model_name=member, model_alias=group.group_name) @@ -12020,7 +12060,10 @@ class Router: ): _settings_to_return[var] = self.lowestlatency_logger.routing_args.json() - _settings_to_return["routing_groups"] = [group.model_dump() for group in self._routing_groups.values()] + _settings_to_return["routing_groups"] = [ + group.model_dump(exclude=frozenset(("model_priorities",)) if group.model_priorities is None else None) + for group in self._routing_groups.values() + ] return _settings_to_return def update_settings(self, **kwargs): @@ -14201,6 +14244,12 @@ class Router: request_kwargs=request_kwargs, ) + if self._is_priority_routing_group(model): + pass_through_deployments = litellm.utils.get_order_filtered_deployments( + pass_through_deployments, + target_order=request_kwargs.pop("_target_order", None) if request_kwargs is not None else None, + ) + if len(pass_through_deployments) == 0: model_ids = self.get_model_ids(model_name=model) _cooldown_time = self.cooldown_cache.get_min_cooldown( diff --git a/litellm/router_utils/add_retry_fallback_headers.py b/litellm/router_utils/add_retry_fallback_headers.py index cbca5880b52..6e07693b7ea 100644 --- a/litellm/router_utils/add_retry_fallback_headers.py +++ b/litellm/router_utils/add_retry_fallback_headers.py @@ -131,6 +131,19 @@ def ensure_response_additional_headers(response: object) -> dict[str, object]: return additional_headers +def apply_response_model_id(response: object, request_metadata: Mapping[str, object] | None) -> None: + if request_metadata is None: + return + model_id: Final = _routing_header_mapping(request_metadata.get("model_info")).get("id") + if not isinstance(model_id, str) or not model_id: + return + hidden_params: Final = get_hidden_params_dict(response, create=isinstance(response, dict)) + if hidden_params.get("model_id"): + return + hidden_params["model_id"] = model_id + _write_hidden_params(response, hidden_params) + + def apply_quality_router_decision_headers( additional_headers: dict[str, object], request_kwargs: object, diff --git a/litellm/router_utils/fallback_event_handlers.py b/litellm/router_utils/fallback_event_handlers.py index 61e0d82e66b..4745e4094e6 100644 --- a/litellm/router_utils/fallback_event_handlers.py +++ b/litellm/router_utils/fallback_event_handlers.py @@ -498,7 +498,12 @@ async def _is_fallback_target_authorized( ) -> bool: access_check: Final = litellm_router.fallback_access_check target: Final = _get_fallback_target_model_group(fallback_entry) - if access_check is None or target is None or target == original_model_group: + if ( + access_check is None + or target is None + or target == original_model_group + or target == get_pre_routing_selection(kwargs) + ): return True if await access_check(model=target, request_kwargs=kwargs, llm_router=litellm_router): return True diff --git a/litellm/router_utils/pre_call_checks/deployment_affinity_check.py b/litellm/router_utils/pre_call_checks/deployment_affinity_check.py index 3b88ac2eb00..edba4c27647 100644 --- a/litellm/router_utils/pre_call_checks/deployment_affinity_check.py +++ b/litellm/router_utils/pre_call_checks/deployment_affinity_check.py @@ -13,7 +13,7 @@ where routing to a consistent deployment is still beneficial. """ import hashlib -from collections.abc import Mapping, Sequence +from collections.abc import Callable, Mapping, Sequence from typing import Any, Final, cast from typing_extensions import ReadOnly, TypedDict @@ -79,6 +79,7 @@ class DeploymentAffinityCheck(CustomLogger): enable_responses_api_affinity: bool, enable_session_id_affinity: bool = False, model_group_affinity_config: dict[str, list[str]] | None = None, + is_priority_group: Callable[[str], bool] | None = None, ): super().__init__() self.cache = cache @@ -87,6 +88,7 @@ class DeploymentAffinityCheck(CustomLogger): self.enable_responses_api_affinity = enable_responses_api_affinity self.enable_session_id_affinity = enable_session_id_affinity self.model_group_affinity_config: dict[str, list[str]] = model_group_affinity_config or {} + self.is_priority_group = is_priority_group def _get_effective_flags(self, model_group: str) -> tuple[bool, bool, bool]: """ @@ -384,6 +386,9 @@ class DeploymentAffinityCheck(CustomLogger): ) return [deployment] + if self.is_priority_group is not None and self.is_priority_group(model): + return typed_healthy_deployments + stable_model_map_key: Final = self._get_stable_model_map_key_from_deployments( healthy_deployments=typed_healthy_deployments ) diff --git a/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py b/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py index 0589e290b47..eabd79f1847 100644 --- a/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py +++ b/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py @@ -4,6 +4,7 @@ Check if prompt caching is valid for a given deployment Route to previously cached model id, if valid """ +from collections.abc import Callable from typing import Final, cast from litellm import verbose_logger @@ -48,8 +49,10 @@ def _get_min_token_count_for_deployments(healthy_deployments: list[dict]) -> int class PromptCachingDeploymentCheck(CustomLogger): - def __init__(self, cache: DualCache): + def __init__(self, cache: DualCache, is_priority_group: Callable[[str], bool] | None = None): + super().__init__() self.cache = cache + self.is_priority_group = is_priority_group async def async_filter_deployments( self, @@ -59,6 +62,8 @@ class PromptCachingDeploymentCheck(CustomLogger): request_kwargs: dict | None = None, parent_otel_span: Span | None = None, ) -> list[dict]: + if self.is_priority_group is not None and self.is_priority_group(model): + return healthy_deployments if request_kwargs is not None and request_kwargs.get("_target_order") is not None: return healthy_deployments diff --git a/litellm/router_utils/routing_groups.py b/litellm/router_utils/routing_groups.py index 772c8bd805c..4438e53ca7d 100644 --- a/litellm/router_utils/routing_groups.py +++ b/litellm/router_utils/routing_groups.py @@ -2,7 +2,20 @@ from collections.abc import Sequence from typing import Final from litellm._logging import verbose_router_logger -from litellm.types.router import RoutingGroup, RoutingStrategy +from litellm.types.router import DeploymentTypedDict, RoutingGroup, RoutingStrategy + + +def apply_routing_group_priority( + group: RoutingGroup, member: str, deployment: DeploymentTypedDict +) -> DeploymentTypedDict: + if group.routing_strategy != "priority" or group.model_priorities is None: + return deployment + prioritized: Final[DeploymentTypedDict] = { + **deployment, + "litellm_params": {**deployment["litellm_params"], "order": group.model_priorities[member]}, + } + return prioritized + VALID_ROUTING_STRATEGIES: Final = ("simple-shuffle", "lar1", *(s.value for s in RoutingStrategy)) @@ -43,10 +56,18 @@ def parse_routing_groups( raise ValueError(f"routing_groups: group names must be unique, duplicate group_name '{min(duplicate_names)}'.") for group in groups: - validate_routing_strategy(group.routing_strategy) + if group.routing_strategy != "priority": + validate_routing_strategy(group.routing_strategy) owners_by_model: Final = tuple( - (model_name, tuple(group.group_name for group in groups if model_name in group.models)) + ( + model_name, + tuple( + group.group_name + for group in groups + if group.routing_strategy != "priority" and model_name in group.models + ), + ) for model_name in dict.fromkeys(model_name for group in groups for model_name in group.models) ) conflicts: Final = tuple( diff --git a/litellm/types/management_endpoints/router_settings_endpoints.py b/litellm/types/management_endpoints/router_settings_endpoints.py index cef180b202a..4d1ac58edb0 100644 --- a/litellm/types/management_endpoints/router_settings_endpoints.py +++ b/litellm/types/management_endpoints/router_settings_endpoints.py @@ -80,6 +80,7 @@ class RouterSettingsField(BaseModel): # Routing strategy descriptions ROUTING_STRATEGY_DESCRIPTIONS: Final[dict[str, str]] = { + "priority": "Routes group calls to the lowest-priority-number available model, with failover to higher numbers. Equal priorities share traffic. Direct member calls keep their existing policy.", "simple-shuffle": "Randomly picks a deployment from the list. Simple and fast.", "least-busy": "Routes to the deployment with the lowest number of ongoing requests.", "latency-based-routing": "Routes to the deployment with the lowest latency over a sliding window.", diff --git a/litellm/types/router.py b/litellm/types/router.py index 5017e7073f7..2353093d435 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -6,7 +6,7 @@ import datetime import enum from collections.abc import Mapping, Sequence from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, ClassVar, Final, Generic, Literal, TypeVar, get_type_hints +from typing import TYPE_CHECKING, Annotated, Any, ClassVar, Final, Generic, Literal, TypeVar, get_type_hints from zoneinfo import ZoneInfo, ZoneInfoNotFoundError import httpx @@ -60,6 +60,25 @@ class RoutingGroup(BaseModel): routing_strategy: str routing_strategy_args: dict | None = None + model_priorities: dict[str, Annotated[int, Field(strict=True, ge=1, le=9007199254740991)]] | None = Field( + default=None, + description="For priority groups, every model's priority. Lower numbers are tried first; equal numbers share traffic.", + ) + + @model_validator(mode="after") + def _validate_model_priorities(self) -> "RoutingGroup": + if self.routing_strategy != "priority": + if self.model_priorities: + raise ValueError("model_priorities requires routing_strategy='priority'") + return self + if not self.models or len(self.models) != len(frozenset(self.models)): + raise ValueError("Priority routing groups require nonempty, distinct models") + if self.model_priorities is None or frozenset(self.model_priorities) != frozenset(self.models): + raise ValueError("model_priorities must contain exactly the group's models") + if self.routing_strategy_args: + raise ValueError("Priority routing groups use model_priorities, not routing_strategy_args") + return self + model_config = ConfigDict(protected_namespaces=()) diff --git a/tests/test_litellm/proxy/management_endpoints/test_router_settings_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_router_settings_endpoints.py index 3fcda310435..ec1af0518c6 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_router_settings_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_router_settings_endpoints.py @@ -6,20 +6,22 @@ Tests the GET endpoints for router settings and router fields. from collections.abc import Mapping from typing import Any, Final -from unittest.mock import AsyncMock, MagicMock, patch import pytest from fastapi.testclient import TestClient - from litellm.proxy import proxy_server from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.config_resolvers import SettingsStore from litellm.proxy.management_endpoints.router_settings_endpoints import ( + RouterFieldsResponse, + RouterSettingsResponse, + get_router_fields, get_router_settings, ) -from litellm.proxy.config_resolvers import SettingsStore from litellm.proxy.proxy_server import app from litellm.router import Router +from litellm.types.router import RoutingGroup client = TestClient(app) @@ -157,3 +159,91 @@ class TestRouterSettingsEndpoints: rg_field = next(f for f in response.fields if f.field_name == "routing_groups") assert rg_field.field_value == groups + + @pytest.mark.asyncio + @pytest.mark.parametrize("metadata_only", (True, False)) + async def test_priority_is_advertised_for_groups_only( + self, monkeypatch: pytest.MonkeyPatch, metadata_only: bool + ) -> None: + monkeypatch.setattr(proxy_server, "llm_router", None) + monkeypatch.setattr( + proxy_server, + "proxy_config", + _StubProxyConfig(SettingsStore("router_settings"), {}), + ) + admin_user: Final = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-test" + ) + + response: Final[RouterFieldsResponse | RouterSettingsResponse] = ( + await get_router_fields(user_api_key_dict=admin_user) + if metadata_only + else await get_router_settings(user_api_key_dict=admin_user) + ) + + global_options: Final = next( + field.options + for field in response.fields + if field.field_name == "routing_strategy" + ) + assert global_options is not None + assert "priority" not in global_options + assert response.model_dump(mode="json")["routing_group_strategies"] == [*global_options, "priority"] + + @pytest.mark.asyncio + @pytest.mark.parametrize("from_config", (False, True)) + async def test_settings_retains_explicit_model_priorities( + self, monkeypatch: pytest.MonkeyPatch, from_config: bool + ) -> None: + group: Final = RoutingGroup( + group_name="ordered-chat", + models=["primary", "backup"], + routing_strategy="priority", + model_priorities={"primary": 1, "backup": 2}, + ) + llm_router: Final = Router( + model_list=[ + { + "model_name": model, + "litellm_params": { + "model": "openai/gpt-5.4-nano", + "api_key": "sk-test", + }, + } + for model in group.models + ], + routing_groups=[group], + ) + expected: Final = [ + { + "group_name": "ordered-chat", + "models": ["primary", "backup"], + "routing_strategy": "priority", + "routing_strategy_args": None, + "model_priorities": ( + {"primary": 8, "backup": 3} + if from_config + else {"primary": 1, "backup": 2} + ), + } + ] + config: Final = {"routing_groups": expected} if from_config else {} + store: Final = SettingsStore("router_settings") + store.load_yaml(config) + monkeypatch.setattr(proxy_server, "llm_router", llm_router) + monkeypatch.setattr( + proxy_server, "proxy_config", _StubProxyConfig(store, config) + ) + admin_user: Final = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-test" + ) + + response: Final[RouterSettingsResponse] = await get_router_settings( + user_api_key_dict=admin_user + ) + + assert response.current_values["routing_groups"] == expected + groups_field: Final = next( + field for field in response.fields if field.field_name == "routing_groups" + ) + assert groups_field.field_value == expected diff --git a/tests/test_litellm/router_strategy/test_router_routing_groups.py b/tests/test_litellm/router_strategy/test_router_routing_groups.py index 506563a82fb..425f68dda18 100644 --- a/tests/test_litellm/router_strategy/test_router_routing_groups.py +++ b/tests/test_litellm/router_strategy/test_router_routing_groups.py @@ -9,7 +9,8 @@ import asyncio import datetime import time import uuid -from collections.abc import Callable +from collections.abc import Callable, Mapping +from typing import Final, Literal from unittest.mock import patch import pytest @@ -18,7 +19,7 @@ from pydantic import ValidationError import litellm from litellm import Router from litellm.integrations.custom_logger import CustomLogger -from litellm.types.router import RoutingGroup, RoutingStrategy +from litellm.types.router import DeploymentTypedDict, FallbackAccessCheck, RoutingGroup, RoutingStrategy from litellm.utils import Rules, function_setup @@ -1676,3 +1677,475 @@ async def test_group_call_429_cools_down_member_across_retries(): ) cooldown_ids = await _call_and_get_cooldowns(router, "quality") assert "deploy-3" in cooldown_ids + + +def _priority_group( + name: str = "priority-group", primary: int = 1, backup: int = 2 +) -> RoutingGroup: + return RoutingGroup.model_validate({ + "group_name": name, + "models": ["filtered-model", "other-model"], + "routing_strategy": "priority", + "model_priorities": {"filtered-model": primary, "other-model": backup}, + }) + + +def _priority_deployments( + primary_response: str = "primary", primary_blocked: bool = False, backup_response: str = "backup" +) -> list[DeploymentTypedDict]: + return [ + { + **deployment, + "litellm_params": { + **deployment["litellm_params"], + "mock_response": ( + primary_response if deployment["model_name"] == "filtered-model" else backup_response + ), + "order": 10 if deployment["model_name"] == "filtered-model" else 1, + }, + "model_info": { + **deployment["model_info"], + "blocked": primary_blocked and deployment["model_name"] == "filtered-model", + }, + } + for deployment in _model_list() + ] + + +def test_priority_group_affinity_scope_follows_group_aliases_and_settings_reload() -> None: + router: Final = Router( + model_list=_priority_deployments(), + routing_groups=[ + _priority_group(), + RoutingGroup(group_name="legacy", models=["filtered-model"], routing_strategy="simple-shuffle"), + ], + model_group_alias={"priority-alias": "priority-group"}, + ) + requested_models: Final = ( + "priority-group", "priority-alias", "filtered-model", "other-model", "legacy", "missing" + ) + assert tuple(router._is_priority_routing_group(model) for model in requested_models) == ( + True, True, False, False, False, False + ) + + router.update_settings(routing_groups=[{ + "group_name": "priority-group", + "models": ["filtered-model", "other-model"], + "routing_strategy": "simple-shuffle", + }]) + assert tuple(router._is_priority_routing_group(model) for model in requested_models) == (False,) * 6 + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("requested_model", ["priority-group", "priority-alias"]) +@pytest.mark.asyncio +async def test_priority_group_always_uses_primary_when_healthy( + asynchronous: bool, requested_model: str +) -> None: + router: Final = Router( + model_list=_priority_deployments(), + routing_groups=[_priority_group()], + model_group_alias={"priority-alias": "priority-group"}, + num_retries=0, + ) + request: Final = {"model": requested_model, "messages": [{"role": "user", "content": "hi"}]} + response: Final = ( + await router.acompletion(**request) if asynchronous else router.completion(**request) + ) + assert response.choices[0].message.content == "primary" + + +@pytest.mark.parametrize("requested_model", ["priority-group", "priority-alias"]) +@pytest.mark.asyncio +async def test_priority_group_fails_over_without_retries_and_leaves_direct_calls_unchanged( + requested_model: str, +) -> None: + router: Final = Router( + model_list=_priority_deployments(primary_response="litellm.RateLimitError"), + routing_groups=[_priority_group()], + model_group_alias={"priority-alias": "priority-group"}, + num_retries=0, + disable_cooldowns=True, + ) + response: Final = await router.acompletion( + model=requested_model, messages=[{"role": "user", "content": "hi"}] + ) + assert response.choices[0].message.content == "backup" + with pytest.raises(litellm.RateLimitError): + await router.acompletion(model="filtered-model", messages=[{"role": "user", "content": "hi"}]) + + +@pytest.mark.asyncio +async def test_opposite_priority_groups_preserve_deployments_and_legacy_member_policy() -> None: + router: Final = Router( + model_list=_priority_deployments(), + routing_strategy="latency-based-routing", + routing_groups=[ + RoutingGroup(group_name="legacy", models=["filtered-model"], routing_strategy="least-busy"), + _priority_group(), + _priority_group("reverse-group", primary=2, backup=1), + ], + ) + before: Final = router.get_model_list(model_name="filtered-model") + forward: Final = await router.acompletion( + model="priority-group", messages=[{"role": "user", "content": "hi"}] + ) + reverse: Final = await router.acompletion( + model="reverse-group", messages=[{"role": "user", "content": "hi"}] + ) + assert (forward.choices[0].message.content, reverse.choices[0].message.content) == ("primary", "backup") + assert router._get_routing_context("filtered-model")[0] == "least-busy" + assert router._get_routing_context("other-model")[0] == "latency-based-routing" + assert router.get_model_list(model_name="filtered-model") == before + assert all(deployment["litellm_params"]["order"] == 10 for deployment in before) + + +@pytest.mark.parametrize("backup_priority, expected", [(1, "backup"), (2, "primary")]) +@pytest.mark.asyncio +async def test_priority_group_weights_select_only_within_the_first_eligible_level( + backup_priority: int, expected: str +) -> None: + router: Final = Router( + model_list=_priority_deployments(), routing_groups=[_priority_group(backup=backup_priority)] + ) + response: Final = await router.acompletion( + model="priority-group", + messages=[{"role": "user", "content": "hi"}], + _router_weights={"priority-group": {"deploy-1": 0, "deploy-2": 0, "deploy-3": 1}}, + ) + assert response.choices[0].message.content == expected + + +@pytest.mark.asyncio +async def test_priority_group_skips_paused_primary_and_returns_to_it_after_recovery() -> None: + router: Final = Router( + model_list=_priority_deployments(primary_blocked=True), routing_groups=[_priority_group()] + ) + paused: Final = await router.acompletion( + model="priority-group", messages=[{"role": "user", "content": "hi"}] + ) + router.set_model_list(_priority_deployments()) + recovered: Final = await router.acompletion( + model="priority-group", messages=[{"role": "user", "content": "hi"}] + ) + assert (paused.choices[0].message.content, recovered.choices[0].message.content) == ("backup", "primary") + + +@pytest.mark.parametrize("controls", [{"disable_fallbacks": True}, {"max_fallbacks": 0}]) +@pytest.mark.asyncio +async def test_priority_group_respects_request_fallback_controls(controls: dict[str, object]) -> None: + router: Final = Router( + model_list=_priority_deployments(primary_response="litellm.RateLimitError"), + routing_groups=[_priority_group()], + num_retries=0, + disable_cooldowns=True, + ) + with pytest.raises(litellm.RateLimitError): + await router.acompletion( + model="priority-group", messages=[{"role": "user", "content": "hi"}], **controls + ) + + +@pytest.mark.parametrize( + "priorities", + [None, {}, {"filtered-model": 1}, {"filtered-model": 1, "other-model": 2, "extra": 3}], +) +def test_priority_group_requires_exact_member_priorities(priorities: object) -> None: + with pytest.raises(ValidationError): + RoutingGroup.model_validate({ + "group_name": "priority-group", + "models": ["filtered-model", "other-model"], + "routing_strategy": "priority", + "model_priorities": priorities, + }) + + +@pytest.mark.parametrize("priority", [True, 0, -1, 1.5, "1", 9007199254740992]) +def test_priority_group_rejects_invalid_priority_values(priority: object) -> None: + with pytest.raises(ValidationError): + RoutingGroup.model_validate({ + "group_name": "priority-group", + "models": ["filtered-model"], + "routing_strategy": "priority", + "model_priorities": {"filtered-model": priority}, + }) + + +@pytest.mark.parametrize("models", [[], ["filtered-model", "filtered-model"]]) +def test_priority_group_requires_nonempty_unique_members(models: list[str]) -> None: + with pytest.raises(ValidationError): + RoutingGroup.model_validate({ + "group_name": "priority-group", + "models": models, + "routing_strategy": "priority", + "model_priorities": {model: 1 for model in models}, + }) + + +@pytest.mark.parametrize( + "changes", [{"routing_strategy": "simple-shuffle"}, {"routing_strategy_args": {"ttl": 60}}] +) +def test_priority_group_rejects_conflicting_strategy_settings(changes: dict[str, object]) -> None: + with pytest.raises(ValidationError): + RoutingGroup.model_validate({ + "group_name": "priority-group", + "models": ["filtered-model"], + "routing_strategy": "priority", + "model_priorities": {"filtered-model": 1}, + **changes, + }) + + +@pytest.mark.parametrize("group_name", ["filtered-model", "priority-alias"]) +def test_priority_group_rejects_names_shadowed_by_a_model_or_alias(group_name: str) -> None: + with pytest.raises(ValueError, match=r"shadow|collid|conflict"): + Router( + model_list=_priority_deployments(), + routing_groups=[_priority_group(name=group_name)], + model_group_alias={"priority-alias": "filtered-model"}, + ) + + +def test_priority_is_rejected_as_a_top_level_strategy() -> None: + with pytest.raises(ValueError, match="routing_strategy"): + _build_router(routing_strategy="priority") + + +@pytest.mark.asyncio +async def test_priority_group_settings_roundtrip_replaces_the_order() -> None: + router: Final = Router(model_list=_priority_deployments(), routing_groups=[_priority_group()]) + replacement: Final = _priority_group(primary=9007199254740991, backup=1) + router.update_settings(routing_groups=[replacement.model_dump()]) + stored: Final = router.get_settings()["routing_groups"] + assert stored == [replacement.model_dump()] + response: Final = await router.acompletion( + model="priority-group", messages=[{"role": "user", "content": "hi"}] + ) + assert response.choices[0].message.content == "backup" + + +def _priority_auto_router_deployment() -> DeploymentTypedDict: + return { + "model_name": "smart-router", + "litellm_params": { + "model": "auto_router/complexity_router", + "complexity_router_config": { + "classifier_type": "heuristic", + "adaptive": False, + "tiers": { + tier: "priority-group" + for tier in ("SIMPLE", "MEDIUM", "COMPLEX", "REASONING") + }, + }, + "complexity_router_default_model": "filtered-model", + }, + } + + +@pytest.mark.asyncio +async def test_auto_router_selected_priority_group_fails_over_inside_the_group() -> None: + router: Final = Router( + model_list=[ + *_priority_deployments(primary_response="litellm.RateLimitError"), + _priority_auto_router_deployment(), + ], + routing_groups=[_priority_group()], + num_retries=0, + disable_cooldowns=True, + ) + response: Final = await router.acompletion( + model="smart-router", messages=[{"role": "user", "content": "hi"}] + ) + assert response.choices[0].message.content == "backup" + + +@pytest.mark.parametrize("endpoint_request", [{"input": "hi"}, {"messages": [{"role": "user", "content": "hi"}]}]) +@pytest.mark.asyncio +async def test_priority_generic_fallback_keeps_routing_controls_out_of_provider_kwargs( + endpoint_request: dict[str, object], +) -> None: + async def provider(model: str, **provider_kwargs: object) -> str: + assert "_target_order" not in provider_kwargs + assert "model_priorities" not in provider_kwargs + model_info: Final = provider_kwargs["model_info"] + assert isinstance(model_info, dict) + if model_info["id"] != "deploy-3": + raise litellm.RateLimitError(message="primary refused", model=model, llm_provider="openai") + return "backup" + + router: Final = Router( + model_list=_priority_deployments(), + routing_groups=[_priority_group()], + num_retries=0, + disable_cooldowns=True, + ) + response: Final = await router._ageneric_api_call_with_fallbacks( + model="priority-group", original_function=provider, **endpoint_request + ) + assert response == "backup" + + +@pytest.mark.parametrize("provider_model_id", [None, "resolved-downstream-id"]) +@pytest.mark.asyncio +async def test_priority_generic_fallback_dict_preserves_served_model_id(provider_model_id: str | None) -> None: + async def provider(model: str, **provider_kwargs: object) -> dict[str, object]: + model_info: Final = provider_kwargs["model_info"] + assert isinstance(model_info, dict) + if model_info["id"] == "deploy-1": + raise litellm.NotFoundError(message="primary missing", model=model, llm_provider="openai") + assert model_info["id"] == "deploy-3" + return { + "content": "backup", + **({"_hidden_params": {"model_id": provider_model_id}} if provider_model_id is not None else {}), + } + + router: Final = Router( + model_list=_priority_deployments()[::2], + routing_groups=[_priority_group()], + num_retries=0, + disable_cooldowns=True, + ) + outer_metadata: Final[dict[str, object]] = {} + response: Final = await router._ageneric_api_call_with_fallbacks( + model="priority-group", + original_function=provider, + messages=[{"role": "user", "content": "hi"}], + litellm_metadata=outer_metadata, + ) + primary_info: Final = outer_metadata["model_info"] + assert isinstance(primary_info, dict) + assert primary_info["id"] == "deploy-1" + assert response["content"] == "backup" + assert response["_hidden_params"]["model_id"] == (provider_model_id or "deploy-3") + assert response["_hidden_params"]["additional_headers"]["x-litellm-attempted-fallbacks"] == 1 + + +@pytest.mark.asyncio +async def test_priority_group_ignores_cached_backup_affinity() -> None: + from litellm.router_utils.prompt_caching_cache import PromptCachingCache + + router: Final = Router( + model_list=_priority_deployments(), + routing_groups=[_priority_group()], + optional_pre_call_checks=["prompt_caching"], + ) + messages: Final = [{ + "role": "user", + "content": [{"type": "text", "text": "word " * 5000, "cache_control": {"type": "ephemeral"}}], + }] + cache: Final = PromptCachingCache(cache=router.cache) + await cache.async_add_model_id(model_id="deploy-3", messages=messages, tools=None) + assert await cache.async_get_model_id(messages=messages, tools=None) == {"model_id": "deploy-3"} + response: Final = await router.acompletion(model="priority-group", messages=messages) + assert response.choices[0].message.content == "primary" + + +@pytest.mark.asyncio +async def test_priority_group_preserves_responses_continuity_on_a_backup() -> None: + from litellm.responses.utils import ResponsesAPIRequestUtils + + router: Final = Router( + model_list=_priority_deployments(), + routing_groups=[_priority_group()], + optional_pre_call_checks=["responses_api_deployment_check"], + ) + previous_response_id: Final = ResponsesAPIRequestUtils._build_responses_api_response_id( + custom_llm_provider="openai", model_id="deploy-3", response_id="resp-prior" + ) + deployment: Final = await router.async_get_available_deployment( + model="priority-group", input="continue", request_kwargs={"previous_response_id": previous_response_id} + ) + assert deployment["model_info"]["id"] == "deploy-3" + + +def _recording_fallback_gate(allowed: frozenset[str], checked: list[str]) -> FallbackAccessCheck: + async def check(*, model: str, request_kwargs: Mapping[str, object], llm_router: Router) -> bool: + checked.append(model) + return model in allowed + + return check + + +@pytest.mark.parametrize("budget_admits_group", [False, True]) +@pytest.mark.asyncio +async def test_auto_priority_advance_preserves_access_scope_and_checks_paid_group_budget( + budget_admits_group: bool, +) -> None: + access_checks: Final[list[str]] = [] + budget_checks: Final[list[str]] = [] + router: Final = Router( + model_list=[ + *_priority_deployments(primary_response="litellm.RateLimitError"), + _priority_auto_router_deployment(), + ], + routing_groups=[_priority_group()], + num_retries=0, + disable_cooldowns=True, + fallback_access_check=_recording_fallback_gate(frozenset({"smart-router"}), access_checks), + fallback_budget_check=_recording_fallback_gate( + frozenset({"priority-group"}) if budget_admits_group else frozenset(), budget_checks + ), + ) + if budget_admits_group: + response: Final = await router.acompletion( + model="smart-router", messages=[{"role": "user", "content": "hi"}] + ) + assert response.choices[0].message.content == "backup" + else: + with pytest.raises(litellm.RateLimitError): + await router.acompletion(model="smart-router", messages=[{"role": "user", "content": "hi"}]) + assert access_checks == [] + assert budget_checks == ["priority-group"] + + +@pytest.mark.asyncio +async def test_auto_priority_group_exhaustion_still_checks_external_fallback_access() -> None: + access_checks: Final[list[str]] = [] + router: Final = Router( + model_list=[ + *_priority_deployments( + primary_response="litellm.RateLimitError", backup_response="litellm.RateLimitError" + ), + _priority_auto_router_deployment(), + { + **_model_list()[2], + "model_name": "external", + "litellm_params": {**_model_list()[2]["litellm_params"], "mock_response": "external"}, + "model_info": {"id": "external-deployment"}, + }, + ], + routing_groups=[_priority_group()], + fallbacks=[{"priority-group": ["external"]}], + num_retries=0, + disable_cooldowns=True, + fallback_access_check=_recording_fallback_gate(frozenset({"smart-router"}), access_checks), + ) + with pytest.raises(litellm.RateLimitError): + await router.acompletion(model="smart-router", messages=[{"role": "user", "content": "hi"}]) + assert access_checks and set(access_checks) == {"external"} + + +@pytest.mark.parametrize("check_kind", ["access", "budget"]) +@pytest.mark.parametrize("metadata_bucket", ["metadata", "litellm_metadata"]) +@pytest.mark.asyncio +async def test_caller_cannot_spoof_a_priority_group_to_bypass_fallback_gates( + check_kind: Literal["access", "budget"], metadata_bucket: str, +) -> None: + checked: Final[list[str]] = [] + check: Final = _recording_fallback_gate(frozenset({"filtered-model"}), checked) + router: Final = Router( + model_list=_priority_deployments(primary_response="litellm.RateLimitError"), + routing_groups=[_priority_group()], + fallbacks=[{"filtered-model": ["priority-group"]}], + num_retries=0, + disable_cooldowns=True, + fallback_access_check=check if check_kind == "access" else None, + fallback_budget_check=check if check_kind == "budget" else None, + ) + with pytest.raises(litellm.RateLimitError): + await router.acompletion( + model="filtered-model", + messages=[{"role": "user", "content": "hi"}], + **{metadata_bucket: {"pre_routing_selected_model": "priority-group"}}, + ) + assert checked == ["priority-group"] diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/router/useRouterFields.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/router/useRouterFields.ts index b5adb0994e6..bb37b559cb6 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/router/useRouterFields.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/router/useRouterFields.ts @@ -16,6 +16,7 @@ export interface RouterSettingsField { export interface RouterFieldsResponse { fields: RouterSettingsField[]; routing_strategy_descriptions: Record; + routing_group_strategies?: string[]; } const routerFieldsKeys = createQueryKeys("routerFields"); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/routingGroups/useRoutingGroups.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/routingGroups/useRoutingGroups.test.ts new file mode 100644 index 00000000000..e5f7814c7c8 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/routingGroups/useRoutingGroups.test.ts @@ -0,0 +1,65 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { act, renderHook, waitFor } from "@testing-library/react"; +import { createElement, type ReactNode } from "react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { getRouterSettingsCall, setCallbacksCall } from "@/components/networking"; +import type { RoutingGroup } from "@/components/routing_groups/types"; +import { useRoutingGroups, useSaveRoutingGroups } from "./useRoutingGroups"; + +vi.mock("@/components/networking", () => ({ + getRouterSettingsCall: vi.fn(), + setCallbacksCall: vi.fn(), +})); + +vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ + default: () => ({ accessToken: "test-token", userId: "admin", userRole: "Admin" }), +})); + +const createWrapper = () => { + const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + return function QueryWrapper({ children }: { children: ReactNode }) { + return createElement(QueryClientProvider, { client }, children); + }; +}; + +describe("routing group settings", () => { + beforeEach(() => vi.clearAllMocks()); + + it.each([ + { metadata: ["simple-shuffle", "priority"], expected: ["simple-shuffle", "priority"] }, + { metadata: undefined, expected: ["simple-shuffle"] }, + ])("uses the advertised group strategies and supports older gateways", async ({ metadata, expected }) => { + vi.mocked(getRouterSettingsCall).mockResolvedValue({ + fields: [{ field_name: "routing_strategy", options: ["simple-shuffle"] }], + routing_group_strategies: metadata, + current_values: {}, + }); + const { result } = renderHook(() => useRoutingGroups(), { wrapper: createWrapper() }); + + await waitFor(() => expect(result.current.data?.availableStrategies).toEqual(expected)); + }); + + it("loads and saves membership priorities through the existing router settings endpoint", async () => { + const groups: RoutingGroup[] = [ + { + group_name: "preferred-chat", + models: ["preferred", "backup"], + routing_strategy: "priority", + model_priorities: { preferred: 1, backup: 4 }, + }, + ]; + vi.mocked(getRouterSettingsCall).mockResolvedValue({ current_values: { routing_groups: groups } }); + vi.mocked(setCallbacksCall).mockResolvedValue({}); + const { result } = renderHook(() => ({ query: useRoutingGroups(), save: useSaveRoutingGroups() }), { + wrapper: createWrapper(), + }); + + await waitFor(() => expect(result.current.query.data?.routingGroups).toEqual(groups)); + await act(async () => { + await result.current.save.mutateAsync(groups); + }); + + expect(setCallbacksCall).toHaveBeenCalledWith("test-token", { router_settings: { routing_groups: groups } }); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/routingGroups/useRoutingGroups.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/routingGroups/useRoutingGroups.ts index 71d27d2017c..5ed6fa6a594 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/routingGroups/useRoutingGroups.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/routingGroups/useRoutingGroups.ts @@ -19,11 +19,12 @@ const fetchRoutingGroups = async (accessToken: string): Promise f?.field_name === "routing_strategy"); + const groupStrategies: unknown = data?.routing_group_strategies ?? routingStrategyField?.options; return { routingGroups: Array.isArray(currentValues.routing_groups) ? currentValues.routing_groups : [], routingStrategy: currentValues.routing_strategy ?? null, - availableStrategies: Array.isArray(routingStrategyField?.options) ? routingStrategyField.options : [], + availableStrategies: Array.isArray(groupStrategies) ? groupStrategies : [], }; }; diff --git a/ui/litellm-dashboard/src/components/routing_groups/RoutingGroupModal.test.tsx b/ui/litellm-dashboard/src/components/routing_groups/RoutingGroupModal.integration.test.tsx similarity index 67% rename from ui/litellm-dashboard/src/components/routing_groups/RoutingGroupModal.test.tsx rename to ui/litellm-dashboard/src/components/routing_groups/RoutingGroupModal.integration.test.tsx index e376c551923..4a9a05c1023 100644 --- a/ui/litellm-dashboard/src/components/routing_groups/RoutingGroupModal.test.tsx +++ b/ui/litellm-dashboard/src/components/routing_groups/RoutingGroupModal.integration.test.tsx @@ -1,4 +1,5 @@ import userEvent from "@testing-library/user-event"; +import { fireEvent } from "@testing-library/react"; import { describe, expect, it, vi } from "vitest"; import { renderWithProviders, screen } from "@/../tests/test-utils"; @@ -6,7 +7,7 @@ import { renderWithProviders, screen } from "@/../tests/test-utils"; import RoutingGroupModal from "./RoutingGroupModal"; import type { RoutingGroup } from "./types"; -const STRATEGIES = ["simple-shuffle", "latency-based-routing", "usage-based-routing"]; +const STRATEGIES = ["simple-shuffle", "latency-based-routing", "usage-based-routing", "priority"]; const MODEL_OPTIONS = ["gpt-4o", "claude-sonnet", "gemini-pro"]; const STRATEGY_DESCRIPTIONS = { "simple-shuffle": "Spreads requests evenly across the group." }; @@ -40,6 +41,14 @@ const STORED_GROUP_NULL_ARGS: RoutingGroup = { routing_strategy_args: null, }; +const STORED_PRIORITY_GROUP: RoutingGroup = { + group_name: "preferred-chat", + models: ["gpt-4o", "claude-sonnet"], + routing_strategy: "priority", + routing_strategy_args: null, + model_priorities: { "gpt-4o": 3, "claude-sonnet": 7 }, +}; + const EXPECTED_NULL_ARGS_PAYLOAD: RoutingGroup = { group_name: "already-taken", models: ["gpt-4o"], @@ -98,6 +107,138 @@ const save = async (user: ReturnType, name: string) => await user.click(screen.getByRole("button", { name })); describe("RoutingGroupModal", () => { + it("creates a priority group with editable defaults and members already used by another group", async () => { + const user = userEvent.setup(); + const { onSubmit } = renderModal({ groupNameByModel: { "gpt-4o": "legacy-group" } }); + + fireEvent.change(screen.getByLabelText("Group Name"), { target: { value: "preferred-chat" } }); + await pickModels(user, "gpt-4o", "claude-sonnet"); + await pickStrategy(user, "Priority"); + + expect(screen.getByLabelText("Priority for gpt-4o")).toHaveValue("1"); + expect(screen.getByLabelText("Priority for claude-sonnet")).toHaveValue("2"); + fireEvent.change(screen.getByLabelText("Priority for claude-sonnet"), { target: { value: "1" } }); + await save(user, "Create Group"); + + const expected: RoutingGroup = { + group_name: "preferred-chat", + models: ["gpt-4o", "claude-sonnet"], + routing_strategy: "priority", + routing_strategy_args: null, + model_priorities: { "gpt-4o": 1, "claude-sonnet": 1 }, + }; + expect(onSubmit).toHaveBeenCalledWith(expected); + }); + + it("preserves explicit priorities through edit for model names that are dictionary keys", async () => { + const user = userEvent.setup(); + const stored: RoutingGroup = { + group_name: "preferred-chat", + models: ["provider/model.v1", "constructor", "__proto__"], + routing_strategy: "priority", + routing_strategy_args: null, + model_priorities: Object.fromEntries([ + ["__proto__", 3], + ["constructor", 8], + ["provider/model.v1", 2], + ]), + }; + const { onSubmit } = renderModal({ mode: "edit", initialValue: stored, modelOptions: stored.models }); + + expect(screen.getByLabelText("Priority for provider/model.v1")).toHaveValue("2"); + expect(screen.getByLabelText("Priority for constructor")).toHaveValue("8"); + await save(user, "Save Changes"); + + expect(onSubmit).toHaveBeenCalledWith(stored); + }); + + it.each([ + { catalog: "omits Priority", strategies: ["simple-shuffle"] }, + { catalog: "includes Priority", strategies: STRATEGIES }, + ])("keeps unsaved priorities when switching away and back while the catalog $catalog", async ({ strategies }) => { + const user = userEvent.setup(); + const { onSubmit } = renderModal({ + mode: "edit", + initialValue: STORED_PRIORITY_GROUP, + availableStrategies: strategies, + }); + + fireEvent.change(screen.getByLabelText("Priority for gpt-4o"), { target: { value: "9" } }); + await pickStrategy(user, "simple-shuffle"); + expect(screen.queryByLabelText("Priority for gpt-4o")).not.toBeInTheDocument(); + await pickStrategy(user, "Priority"); + expect(screen.getByLabelText("Priority for gpt-4o")).toHaveValue("9"); + expect(screen.getByLabelText("Priority for claude-sonnet")).toHaveValue("7"); + await save(user, "Save Changes"); + + expect(onSubmit).toHaveBeenCalledWith({ + ...STORED_PRIORITY_GROUP, + model_priorities: { "gpt-4o": 9, "claude-sonnet": 7 }, + }); + }); + + it("offers only the backend catalog when creating a group", async () => { + const user = userEvent.setup(); + renderModal({ initialValue: STORED_PRIORITY_GROUP, availableStrategies: ["simple-shuffle"] }); + + await user.click(screen.getByLabelText("Routing Strategy")); + + expect(await screen.findByRole("option", { name: "simple-shuffle" })).toBeInTheDocument(); + expect(screen.queryByRole("option", { name: "Priority" })).not.toBeInTheDocument(); + }); + + it("shows invalid stored priorities and lets the user repair them before saving", async () => { + const user = userEvent.setup(); + const { onSubmit } = renderModal({ + mode: "edit", + initialValue: { + group_name: "preferred-chat", + models: ["gpt-4o"], + routing_strategy: "priority", + model_priorities: { "gpt-4o": 0, unused: 9 }, + }, + }); + + expect(screen.getByLabelText("Priority for gpt-4o")).toHaveValue("0"); + expect(screen.getByText("Model is not selected")).toBeInTheDocument(); + await save(user, "Save Changes"); + expect(onSubmit).not.toHaveBeenCalled(); + await user.click(screen.getByRole("button", { name: "Remove priority for unused" })); + await save(user, "Save Changes"); + expect(await screen.findByText("Priorities must be whole numbers from 1 to 9007199254740991")).toBeInTheDocument(); + fireEvent.change(screen.getByLabelText("Priority for gpt-4o"), { target: { value: "4" } }); + await save(user, "Save Changes"); + + const expected: RoutingGroup = { + group_name: "preferred-chat", + models: ["gpt-4o"], + routing_strategy: "priority", + routing_strategy_args: null, + model_priorities: { "gpt-4o": 4 }, + }; + expect(onSubmit).toHaveBeenCalledWith(expected); + }); + + it("restores the legacy ownership restriction when switching away from priority", async () => { + const user = userEvent.setup(); + const { onSubmit } = renderModal({ + mode: "edit", + initialValue: { + group_name: "preferred-chat", + models: ["gpt-4o"], + routing_strategy: "priority", + model_priorities: { "gpt-4o": 1 }, + }, + groupNameByModel: { "gpt-4o": "legacy-group" }, + }); + + await pickStrategy(user, "simple-shuffle"); + await save(user, "Save Changes"); + + expect(await screen.findByText(/Already claimed: gpt-4o/)).toBeInTheDocument(); + expect(onSubmit).not.toHaveBeenCalled(); + }); + it("submits an untouched edit of a group whose stored arguments are null", async () => { const user = userEvent.setup(); const { onSubmit } = renderModal({ mode: "edit", initialValue: STORED_GROUP_NULL_ARGS }); diff --git a/ui/litellm-dashboard/src/components/routing_groups/RoutingGroupModal.tsx b/ui/litellm-dashboard/src/components/routing_groups/RoutingGroupModal.tsx index 1057cc6ca16..01bdbae4bf2 100644 --- a/ui/litellm-dashboard/src/components/routing_groups/RoutingGroupModal.tsx +++ b/ui/litellm-dashboard/src/components/routing_groups/RoutingGroupModal.tsx @@ -26,6 +26,7 @@ import { STRATEGIES_WITH_ARGS, argsForStrategy, buildRoutingGroupPayload, + prioritiesForModels, toRoutingGroupFormValues, } from "./routingGroupPayload"; import type { RoutingGroup } from "./types"; @@ -65,7 +66,14 @@ const RoutingGroupModal: React.FC = ({ saving, }) => { const modelsAnchor = useComboboxAnchor(); - const strategyItems = availableStrategies.map((strategy) => ({ label: strategy, value: strategy })); + const selectableStrategies = + mode === "edit" && initialValue + ? Array.from(new Set([...availableStrategies, initialValue.routing_strategy])) + : availableStrategies; + const strategyItems = selectableStrategies.map((strategy) => ({ + label: strategy === "priority" ? "Priority" : strategy, + value: strategy, + })); const reservedNames = useMemo(() => { const others = existingGroupNames.filter((n) => n !== initialValue?.group_name); @@ -80,19 +88,18 @@ const RoutingGroupModal: React.FC = ({ .min(1, "Group name is required") .max(GROUP_NAME_MAX_LENGTH, `Must be ${GROUP_NAME_MAX_LENGTH} characters or fewer`) .refine((value) => !reservedNames.has(value.toLowerCase()), "A group with this name already exists"), - models: z - .array(z.string()) - .min(1, "Select at least one model") - .superRefine((models, ctx) => { - const conflict = modelConflictError(models, groupNameByModel); - if (conflict !== null) { - ctx.addIssue({ code: "custom", message: conflict }); - } - }), + models: z.array(z.string()).min(1, "Select at least one model"), routing_strategy: z.string().min(1, "Strategy is required"), routing_strategy_args: z.string(), + model_priorities: z.array(z.object({ model: z.string(), priority: z.string() })), }; - return z.object(shape); + return z.object(shape).superRefine((values, ctx) => { + if (values.routing_strategy === "priority") return; + const conflict = modelConflictError(values.models, groupNameByModel); + if (conflict !== null) { + ctx.addIssue({ code: "custom", message: conflict, path: ["models"] }); + } + }); }, [reservedNames, groupNameByModel]); const form = useZodForm(schema, { defaultValues: toRoutingGroupFormValues(initialValue, availableStrategies) }); @@ -102,11 +109,12 @@ const RoutingGroupModal: React.FC = ({ }, [open, initialValue, availableStrategies, form]); const selectedStrategy = useWatch({ control: form.control, name: "routing_strategy" }); + const selectedModels = useWatch({ control: form.control, name: "models" }); const handleSubmit = async (values: z.infer) => { const payload = buildRoutingGroupPayload(values); if (!payload.ok) { - form.setError("routing_strategy_args", { message: payload.argsError }); + form.setError(payload.field, { message: payload.message }); return; } await onSubmit(payload.group); @@ -126,7 +134,7 @@ const RoutingGroupModal: React.FC = ({ control={form.control} name="group_name" label="Group Name" - description="Use this name as the model in API calls — LiteLLM routes the request to one of the group's models." + description="Use this name as the model in API calls. LiteLLM routes the request to one of the group's models." > {({ ref, ...field }) => } @@ -135,10 +143,22 @@ const RoutingGroupModal: React.FC = ({ control={form.control} name="models" label="Models" - description="Models from your model list that this group routes between. A model can only be in one group." + description={ + selectedStrategy === "priority" + ? "Models from your model list that this group routes between. Models can belong to multiple priority groups." + : "Models from your model list that this group routes between. A model can belong to one non-priority group." + } > {({ id, value, onChange, "aria-invalid": ariaInvalid, "aria-describedby": ariaDescribedBy }) => ( - + { + onChange(models); + form.setValue("model_priorities", prioritiesForModels(models, form.getValues("model_priorities"))); + }} + > }> {(selected: string[]) => ( @@ -176,7 +196,11 @@ const RoutingGroupModal: React.FC = ({ control={form.control} name="routing_strategy" label="Routing Strategy" - description={strategyDescriptions[selectedStrategy]} + description={ + selectedStrategy === "priority" + ? "Lower priorities are tried first. Models with the same priority share traffic. Applies only when calling this group." + : strategyDescriptions[selectedStrategy] + } > {({ id, value, onChange, "aria-invalid": ariaInvalid, "aria-describedby": ariaDescribedBy }) => ( + onChange( + value.map((item, itemIndex) => + itemIndex === index ? { ...item, priority: event.target.value } : item, + ), + ) + } + /> + {!selectedModels.includes(entry.model) && ( + + )} + + ))} + + )} + + )} + {STRATEGIES_WITH_ARGS.has(selectedStrategy) && ( = ({ )}

- Models not claimed by an explicit group fall through to the proxy's top-level routing strategy. + {selectedStrategy === "priority" + ? "Direct requests to a member model keep their existing routing behavior." + : "Models outside non-priority groups use the proxy's top-level routing strategy."}

diff --git a/ui/litellm-dashboard/src/components/routing_groups/RoutingGroupUsagePanel.tsx b/ui/litellm-dashboard/src/components/routing_groups/RoutingGroupUsagePanel.tsx index fafea569012..85dd67099d6 100644 --- a/ui/litellm-dashboard/src/components/routing_groups/RoutingGroupUsagePanel.tsx +++ b/ui/litellm-dashboard/src/components/routing_groups/RoutingGroupUsagePanel.tsx @@ -14,7 +14,8 @@ interface RoutingGroupUsagePanelProps { baseUrl: string; } -const exampleModel = (group: RoutingGroup): string => group.models[0] ?? ""; +const exampleModel = (group: RoutingGroup): string => + group.routing_strategy === "priority" ? group.group_name : group.models[0] ?? ""; const buildCurlSnippet = (group: RoutingGroup, baseUrl: string): string => `curl -X POST '${baseUrl}/v1/chat/completions' \\ @@ -69,8 +70,17 @@ export function RoutingGroupUsagePanel({ group, baseUrl }: RoutingGroupUsagePane How routing works for this group

- Callers request any model in the group by name; LiteLLM picks a deployment behind the scenes using the{" "} - {formatStrategyLabel(group.routing_strategy)} strategy. + {group.routing_strategy === "priority" ? ( + <> + Request {group.group_name} to try eligible models in + priority order. Direct requests to a member model keep their existing routing behavior. + + ) : ( + <> + Callers request any model in the group by name; LiteLLM picks a deployment behind the scenes using the{" "} + {formatStrategyLabel(group.routing_strategy)} strategy. + + )}

diff --git a/ui/litellm-dashboard/src/components/routing_groups/RoutingGroupsTable.test.tsx b/ui/litellm-dashboard/src/components/routing_groups/RoutingGroupsTable.integration.test.tsx similarity index 86% rename from ui/litellm-dashboard/src/components/routing_groups/RoutingGroupsTable.test.tsx rename to ui/litellm-dashboard/src/components/routing_groups/RoutingGroupsTable.integration.test.tsx index 6f14b76e2fd..6664ecab623 100644 --- a/ui/litellm-dashboard/src/components/routing_groups/RoutingGroupsTable.test.tsx +++ b/ui/litellm-dashboard/src/components/routing_groups/RoutingGroupsTable.integration.test.tsx @@ -113,6 +113,22 @@ describe("RoutingGroupsTable", () => { expect(panel?.textContent).toContain("gpt-4o"); }); + it("uses the callable group name in every priority routing example", async () => { + const user = userEvent.setup(); + render(); + expect(within(rowFor("prod-group")).getByText("Priority")).toBeInTheDocument(); + await user.click(screen.getByRole("button", { name: "prod-group" })); + + expect(screen.getByRole("tabpanel")).toHaveTextContent('"model": "prod-group"'); + await user.click(screen.getByRole("tab", { name: "Python (OpenAI SDK)" })); + expect(screen.getByRole("tabpanel")).toHaveTextContent('model="prod-group"'); + await user.click(screen.getByRole("tab", { name: "JavaScript (OpenAI SDK)" })); + expect(screen.getByRole("tabpanel")).toHaveTextContent('model: "prod-group"'); + expect( + screen.getByText(/Direct requests to a member model keep their existing routing behavior/), + ).toBeInTheDocument(); + }); + it("should expand only the clicked group", async () => { const user = userEvent.setup(); render(); diff --git a/ui/litellm-dashboard/src/components/routing_groups/index.tsx b/ui/litellm-dashboard/src/components/routing_groups/index.tsx index 17329d0b572..ae2d62ae20f 100644 --- a/ui/litellm-dashboard/src/components/routing_groups/index.tsx +++ b/ui/litellm-dashboard/src/components/routing_groups/index.tsx @@ -46,6 +46,7 @@ const RoutingGroups: React.FC = () => { const availableStrategies = useMemo(() => { if (data?.availableStrategies?.length) return data.availableStrategies; + if (routerFields?.routing_group_strategies?.length) return routerFields.routing_group_strategies; const fromFields = routerFields?.fields?.find((f) => f.field_name === "routing_strategy")?.options; return fromFields ?? []; }, [data?.availableStrategies, routerFields]); @@ -178,8 +179,17 @@ const RoutingGroups: React.FC = () => { Delete routing group?

- Models in {deletingGroup?.group_name} will fall back to the - proxy's top-level routing strategy. This cannot be undone. + {deletingGroup?.routing_strategy === "priority" ? ( + <> + Calls to {deletingGroup.group_name} will stop working. Direct + requests to its member models keep their existing routing behavior. This cannot be undone. + + ) : ( + <> + Models in {deletingGroup?.group_name} will fall back to the + proxy's top-level routing strategy. This cannot be undone. + + )}