feat(router): per-model-group routing strategy via model_info

litellm/router.py

@@ -22,6 +22,7 @@ import weakref
 from collections import defaultdict
 from collections.abc import AsyncGenerator, Callable, Generator, Mapping, Sequence
 from functools import lru_cache
+from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Literal, Optional, TypeVar, Union, cast
 
 import anyio
@@ -714,6 +715,7 @@ class Router:
         self._init_routing_groups(self._routing_groups_input)
         self._override_selectors: dict[str, Any] = {}
         self._override_selectors_lock = threading.Lock()
+        self._warned_model_group_strategy_models: set[str] = set()  # mutable-ok: warn-once misconfig registry
         self.access_groups = None
         ## USAGE TRACKING ##
         if isinstance(litellm._async_success_callback, list):
@@ -1125,18 +1127,58 @@ class Router:
                 )
             return self._override_selectors[strategy]
 
+    _EMPTY_MAPPING: Final[Mapping[str, object]] = MappingProxyType({})
+
+    def _get_model_group_strategy_config(self, model: str) -> tuple[str, Mapping[str, object]] | None:
+        """
+        Reads `model_info.routing_strategy` (+ `routing_strategy_args`) off the
+        deployments of `model`. When deployments of the same model_name disagree,
+        the first deployment in model_list order wins; invalid or conflicting
+        values are reported once per model_name so a bad stored value can never
+        take down that model's traffic. An empty string counts as unset, so a
+        PATCH (which cannot delete a model_info key) can still clear the field.
+        """
+        indices: Final = self.model_name_to_deployment_indices.get(model)
+        if not indices:
+            return None
+        configured: Final = tuple(
+            (
+                self._normalize_strategy(raw) if isinstance(raw, (str, RoutingStrategy)) else None,
+                info.get("routing_strategy_args") or self._EMPTY_MAPPING,
+            )
+            for idx in indices
+            if (raw := (info := self.model_list[idx].get("model_info") or self._EMPTY_MAPPING).get("routing_strategy"))
+        )
+        if not configured:
+            return None
+        valid: Final = tuple(
+            (strategy, args)
+            for strategy, args in configured
+            if strategy is not None and strategy in self._OVERRIDABLE_ROUTING_STRATEGIES
+        )
+        has_invalid: Final = len(valid) < len(configured)
+        has_conflict: Final = len(frozenset(strategy for strategy, _ in valid)) > 1
+        if (has_invalid or has_conflict) and model not in self._warned_model_group_strategy_models:
+            self._warned_model_group_strategy_models.add(model)
+            verbose_router_logger.warning(
+                "model_info.routing_strategy for model_group '%s' has %s; using '%s'. Supported strategies: %s.",
+                model,
+                "an unsupported value" if has_invalid else "conflicting values across deployments",
+                valid[0][0] if valid else self._normalize_strategy(self.routing_strategy),
+                sorted(self._OVERRIDABLE_ROUTING_STRATEGIES),
+            )
+        return valid[0] if valid else None
+
     def _get_routing_context(self, model: str, request_kwargs: dict | None = None) -> tuple[str | None, Any | None]:
         """
         Resolves the routing strategy and selector to use for the given model.
 
-        A per-request `routing_strategy` in `request_kwargs` (forwarded by the
-        proxy from key/team `router_settings`) takes precedence over both the
-        model's routing group and the router's top-level strategy, since it is
-        the most specific expression of caller intent.
-
-        Otherwise every model belongs to exactly one group: an explicit entry
-        from `routing_groups`, or the implicit `"default"` group driven by the
-        router's top-level `routing_strategy` / `routing_strategy_args`.
+        Precedence, most specific first: a per-request `routing_strategy` in
+        `request_kwargs` (forwarded by the proxy from key/team
+        `router_settings`), then `model_info.routing_strategy` on the model's
+        own deployments, then the model's `routing_groups` entry (legacy), then
+        the implicit `"default"` group driven by the router's top-level
+        `routing_strategy` / `routing_strategy_args`.
 
         `self.routing_strategy` may be either a string or a `RoutingStrategy`
         enum member (the constructor accepts both), so it is normalized to a
@@ -1148,6 +1190,21 @@ class Router:
             verbose_router_logger.debug("routing_group=request-override model=%s strategy=%s", model, override)
             return override, self._get_override_strategy_selector(override)
 
+        model_group_config: Final = self._get_model_group_strategy_config(model)
+        if model_group_config is not None:
+            mg_strategy, mg_args = model_group_config
+            verbose_router_logger.debug("routing_group=model-info model=%s strategy=%s", model, mg_strategy)
+            if not mg_args:
+                return mg_strategy, self._get_override_strategy_selector(mg_strategy)
+            selector_key: Final = f"{mg_strategy}|{json.dumps(mg_args, sort_keys=True, default=str)}"
+            with self._override_selectors_lock:
+                if selector_key not in self._override_selectors:
+                    self._override_selectors[selector_key] = self._build_strategy_selector(
+                        strategy=mg_strategy,
+                        routing_strategy_args=mg_args,
+                    )
+                return mg_strategy, self._override_selectors[selector_key]
+
         group_name: Final = self._model_to_group.get(model)
         if group_name is None:
             strategy = self._normalize_strategy(self.routing_strategy)

litellm/types/router.py

@@ -4,6 +4,7 @@ litellm.Router Types - includes RouterConfig, UpdateRouterConfig, ModelInfo etc
 
 import datetime
 import enum
+from collections.abc import Mapping
 from dataclasses import dataclass
 from typing import Any, Final, Generic, Literal, TypeVar, get_type_hints
 
@@ -134,6 +135,9 @@ class ModelInfo(BaseModel):
     base_model: str | None = None  # specify if the base model is azure/gpt-3.5-turbo etc for accurate cost tracking
     tier: Literal["free", "paid"] | None = None
 
+    routing_strategy: str | None = None
+    routing_strategy_args: Mapping[str, object] | None = None
+
     """
     Team Model Specific Fields
     """

tests/test_litellm/router_strategy/test_model_group_routing_strategy.py

@@ -0,0 +1,211 @@
+"""
+Tests for `model_info.routing_strategy` — a per-model-group routing strategy
+declared on the model definition itself. Applies to every request for that
+model_name, sits between the per-request override and legacy `routing_groups`
+in precedence, and never raises on bad stored values.
+"""
+
+import logging
+import os
+import sys
+from unittest.mock import patch
+
+import pytest
+
+sys.path.insert(0, os.path.abspath("../.."))
+
+import litellm
+from litellm import Router
+from litellm.router_strategy.lowest_cost import LowestCostLoggingHandler
+from litellm.router_strategy.lowest_latency import LowestLatencyLoggingHandler
+
+
+def _deployment(model_name, model, deployment_id, model_info=None):
+    return {
+        "model_name": model_name,
+        "litellm_params": {
+            "model": model,
+            "api_key": "sk-test",
+            "api_base": "https://example.invalid",
+        },
+        "model_info": {"id": deployment_id, **(model_info or {})},
+    }
+
+
+def _build_router(model_list, routing_strategy="simple-shuffle", routing_groups=None):
+    return Router(
+        model_list=model_list,
+        routing_strategy=routing_strategy,
+        routing_groups=routing_groups,
+    )
+
+
+def test_model_info_strategy_overrides_top_level():
+    router = _build_router(
+        [
+            _deployment("quality", "openai/gpt-4o", "d1", {"routing_strategy": "cost-based-routing"}),
+            _deployment("quality", "openai/gpt-4o-mini", "d2", {"routing_strategy": "cost-based-routing"}),
+            _deployment("plain", "openai/gpt-4o-mini", "d3"),
+        ]
+    )
+    strategy, selector = router._get_routing_context("quality")
+    assert strategy == "cost-based-routing"
+    assert isinstance(selector, LowestCostLoggingHandler)
+    assert router._get_routing_context("plain") == ("simple-shuffle", None)
+
+
+def test_strategy_on_one_deployment_covers_the_whole_model_group():
+    router = _build_router(
+        [
+            _deployment("quality", "openai/gpt-4o", "d1", {"routing_strategy": "cost-based-routing"}),
+            _deployment("quality", "openai/gpt-4o-mini", "d2"),
+        ]
+    )
+    strategy, _ = router._get_routing_context("quality")
+    assert strategy == "cost-based-routing"
+
+
+def test_request_override_beats_model_info():
+    router = _build_router(
+        [_deployment("quality", "openai/gpt-4o", "d1", {"routing_strategy": "cost-based-routing"})]
+    )
+    strategy, selector = router._get_routing_context("quality", {"routing_strategy": "latency-based-routing"})
+    assert strategy == "latency-based-routing"
+    assert isinstance(selector, LowestLatencyLoggingHandler)
+
+
+def test_model_info_beats_legacy_routing_group():
+    router = _build_router(
+        [
+            _deployment("quality", "openai/gpt-4o", "d1", {"routing_strategy": "cost-based-routing"}),
+            _deployment("grouped-only", "openai/gpt-4o-mini", "d2"),
+        ],
+        routing_groups=[
+            {
+                "group_name": "legacy",
+                "models": ["quality", "grouped-only"],
+                "routing_strategy": "latency-based-routing",
+            }
+        ],
+    )
+    assert router._get_routing_context("quality")[0] == "cost-based-routing"
+    assert router._get_routing_context("grouped-only")[0] == "latency-based-routing"
+
+
+def test_simple_shuffle_in_model_info_overrides_nondefault_top_level():
+    router = _build_router(
+        [_deployment("quality", "openai/gpt-4o", "d1", {"routing_strategy": "simple-shuffle"})],
+        routing_strategy="latency-based-routing",
+    )
+    assert router._get_routing_context("quality") == ("simple-shuffle", None)
+
+
+def test_conflicting_values_first_deployment_wins_and_warns_once(caplog):
+    router = _build_router(
+        [
+            _deployment("quality", "openai/gpt-4o", "d1", {"routing_strategy": "cost-based-routing"}),
+            _deployment("quality", "openai/gpt-4o-mini", "d2", {"routing_strategy": "latency-based-routing"}),
+        ]
+    )
+    with caplog.at_level(logging.WARNING, logger="LiteLLM Router"):
+        assert router._get_routing_context("quality")[0] == "cost-based-routing"
+        assert router._get_routing_context("quality")[0] == "cost-based-routing"
+    conflict_warnings = [r for r in caplog.records if "conflicting values" in r.getMessage()]
+    assert len(conflict_warnings) == 1
+    assert "quality" in conflict_warnings[0].getMessage()
+
+
+def test_empty_string_counts_as_unset_without_warning(caplog):
+    router = _build_router(
+        [_deployment("quality", "openai/gpt-4o", "d1", {"routing_strategy": ""})],
+        routing_strategy="latency-based-routing",
+    )
+    with caplog.at_level(logging.WARNING, logger="LiteLLM Router"):
+        strategy, _ = router._get_routing_context("quality")
+    assert strategy == "latency-based-routing"
+    assert not any("model_info.routing_strategy" in r.getMessage() for r in caplog.records)
+
+
+def test_invalid_value_ignored_with_warning(caplog):
+    router = _build_router(
+        [_deployment("quality", "openai/gpt-4o", "d1", {"routing_strategy": "not-a-strategy"})],
+        routing_strategy="latency-based-routing",
+    )
+    with caplog.at_level(logging.WARNING, logger="LiteLLM Router"):
+        strategy, selector = router._get_routing_context("quality")
+    assert strategy == "latency-based-routing"
+    assert isinstance(selector, LowestLatencyLoggingHandler)
+    assert any("unsupported value" in r.getMessage() for r in caplog.records)
+
+
+def test_selector_shared_across_model_groups_with_identical_config():
+    router = _build_router(
+        [
+            _deployment("a", "openai/gpt-4o", "d1", {"routing_strategy": "latency-based-routing"}),
+            _deployment("b", "openai/gpt-4o-mini", "d2", {"routing_strategy": "latency-based-routing"}),
+            _deployment(
+                "c",
+                "openai/gpt-4o-mini",
+                "d3",
+                {"routing_strategy": "latency-based-routing", "routing_strategy_args": {"ttl": 120}},
+            ),
+        ]
+    )
+    _, selector_a = router._get_routing_context("a")
+    _, selector_b = router._get_routing_context("b")
+    _, selector_c = router._get_routing_context("c")
+    assert selector_a is selector_b
+    assert selector_c is not selector_a
+    assert selector_c.routing_args.ttl == 120
+
+
+def test_reuses_default_selector_when_config_matches_top_level():
+    router = _build_router(
+        [_deployment("quality", "openai/gpt-4o", "d1", {"routing_strategy": "cost-based-routing"})],
+        routing_strategy="cost-based-routing",
+    )
+    _, selector = router._get_routing_context("quality")
+    assert selector is router.lowestcost_logger
+
+
+@pytest.mark.asyncio
+async def test_async_dispatch_uses_model_info_strategy():
+    router = _build_router(
+        [
+            _deployment("quality", "openai/gpt-4o", "d1", {"routing_strategy": "latency-based-routing"}),
+            _deployment("quality", "openai/gpt-4o-mini", "d2"),
+            _deployment("plain", "openai/gpt-4o-mini", "d3"),
+        ]
+    )
+    _, selector = router._get_routing_context("quality")
+
+    with (
+        patch.object(
+            selector,
+            "async_get_available_deployments",
+            wraps=selector.async_get_available_deployments,
+        ) as latency_spy,
+        patch("litellm.router.simple_shuffle", wraps=litellm.router.simple_shuffle) as shuffle_spy,
+    ):
+        await router.async_get_available_deployment(model="quality", request_kwargs={})
+        assert latency_spy.called
+        assert not shuffle_spy.called
+
+        await router.async_get_available_deployment(model="plain", request_kwargs={})
+        assert shuffle_spy.called
+
+
+def test_update_settings_resets_cached_model_group_selectors():
+    router = _build_router(
+        [_deployment("quality", "openai/gpt-4o", "d1", {"routing_strategy": "latency-based-routing"})]
+    )
+    _, selector = router._get_routing_context("quality")
+    assert selector in router._override_selectors.values()
+
+    router.update_settings(routing_strategy="cost-based-routing")
+    assert router._override_selectors == {}
+    assert all(c is not selector for c in litellm.callbacks)
+
+    _, rebuilt = router._get_routing_context("quality")
+    assert isinstance(rebuilt, LowestLatencyLoggingHandler)
+    assert rebuilt is not selector

ui/litellm-dashboard/src/components/add_model/advanced_settings.tsx

@@ -6,6 +6,7 @@ import TextArea from "antd/es/input/TextArea";
 import { InfoCircleOutlined } from "@ant-design/icons";
 import { Team } from "../key_team_helpers/key_list";
 import CacheControlSettings from "./cache_control_settings";
+import { ROUTING_STRATEGY_OPTIONS } from "./routing_strategy_options";
 import VectorStoreSelector from "../vector_store_management/VectorStoreSelector";
 import { Tag } from "../tag_management/types";
 import { formItemValidateJSON } from "../../utils/textUtils";
@@ -169,6 +170,29 @@ const AdvancedSettings: React.FC<AdvancedSettingsProps> = ({
               />
             </Form.Item>
 
+            <Form.Item
+              label={
+                <span>
+                  Routing Strategy{" "}
+                  <Tooltip title="Load-balancing strategy used when this model name has multiple deployments. Applies to the whole model group; if left unset, the router's top-level strategy is used.">
+                    <a
+                      href="https://docs.litellm.ai/docs/routing"
+                      target="_blank"
+                      rel="noopener noreferrer"
+                      onClick={(e) => e.stopPropagation()}
+                    >
+                      <InfoCircleOutlined style={{ marginLeft: "4px" }} />
+                    </a>
+                  </Tooltip>
+                </span>
+              }
+              name="routing_strategy"
+              className="mb-4"
+              help="How requests are spread across this model name's deployments."
+            >
+              <Select allowClear placeholder="Inherit router default" options={[...ROUTING_STRATEGY_OPTIONS]} />
+            </Form.Item>
+
             <Form.Item label="Tags" name="tags" className="mb-4">
               <Select
                 mode="tags"

ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.test.tsx

@@ -56,6 +56,26 @@ describe("prepareModelAddRequest", () => {
     expect(deployment.litellmParamsObj.custom_llm_provider).toBe("petals");
   });
 
+  it("routes routing_strategy into model_info, not litellm_params", async () => {
+    const formValues = {
+      model_mappings: [
+        {
+          public_name: "quality",
+          litellm_model: "bedrock/claude-opus",
+        },
+      ],
+      model_name: "bedrock/claude-opus",
+      routing_strategy: "cost-based-routing",
+    };
+
+    const deployments = await prepareModelAddRequest({ ...formValues }, "token", null);
+
+    expect(deployments).toHaveLength(1);
+    const [deployment] = deployments!;
+    expect(deployment.modelInfoObj.routing_strategy).toBe("cost-based-routing");
+    expect(deployment.litellmParamsObj.routing_strategy).toBeUndefined();
+  });
+
   it("ignores litellm_credential_name inside LiteLLM Params JSON", async () => {
     const formValues = {
       model_mappings: [

ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.tsx

@@ -109,6 +109,8 @@ export const prepareModelAddRequest = async (formValues: Record<string, any>, ac
           modelInfoObj[key] = value;
         } else if (key === "team_id") {
           modelInfoObj["team_id"] = value;
+        } else if (key === "routing_strategy") {
+          modelInfoObj["routing_strategy"] = value;
         } else if (key === "model_access_group") {
           modelInfoObj["access_groups"] = value;
         } else if (key == "mode") {

ui/litellm-dashboard/src/components/add_model/routing_strategy_options.ts

@@ -0,0 +1,13 @@
+export const ROUTING_STRATEGY_OPTIONS = [
+  { value: "simple-shuffle", label: "Simple Shuffle (weighted random)" },
+  { value: "latency-based-routing", label: "Latency-Based (fastest deployment)" },
+  { value: "cost-based-routing", label: "Cost-Based (cheapest deployment)" },
+  { value: "usage-based-routing-v2", label: "Usage-Based v2 (lowest TPM load)" },
+  { value: "usage-based-routing", label: "Usage-Based v1 (lowest TPM load)" },
+  { value: "least-busy", label: "Least-Busy (fewest in-flight requests)" },
+] as const;
+
+export const routingStrategyLabel = (value: string | undefined | null): string => {
+  const match = ROUTING_STRATEGY_OPTIONS.find((o) => o.value === value);
+  return match ? match.label : "Inherit router default";
+};

ui/litellm-dashboard/src/components/model_info_view.tsx

@@ -18,6 +18,7 @@ import {
   Button as TremorButton,
 } from "@tremor/react";
 import { Button, Form, Input, Modal, Select, Tooltip } from "antd";
+import { ROUTING_STRATEGY_OPTIONS, routingStrategyLabel } from "./add_model/routing_strategy_options";
 import VectorStoreSelector from "./vector_store_management/VectorStoreSelector";
 import { CheckIcon, CopyIcon } from "lucide-react";
 import { useEffect, useMemo, useState } from "react";
@@ -427,6 +428,13 @@ export default function ModelInfoView({
             health_check_model: values.health_check_model,
           };
         }
+        const formRoutingStrategy = values.routing_strategy ?? "";
+        if (formRoutingStrategy !== (modelData.model_info?.routing_strategy ?? "")) {
+          updatedModelInfo = {
+            ...updatedModelInfo,
+            routing_strategy: formRoutingStrategy,
+          };
+        }
       } catch (e) {
         NotificationsManager.fromBackend("Invalid JSON in Model Info");
         return;
@@ -800,6 +808,7 @@ export default function ModelInfoView({
                         : undefined,
                     tags: Array.isArray(localModelData.litellm_params?.tags) ? localModelData.litellm_params.tags : [],
                     health_check_model: isWildcardModel ? localModelData.model_info?.health_check_model : null,
+                    routing_strategy: localModelData.model_info?.routing_strategy || "",
                     litellm_credential_name: localModelData.litellm_params?.litellm_credential_name || "",
                     litellm_extra_params: JSON.stringify(
                       Object.fromEntries(
@@ -1069,6 +1078,24 @@ export default function ModelInfoView({
                         )}
                       </div>
 
+                      <div>
+                        <Text className="font-medium">Routing Strategy</Text>
+                        {isEditing ? (
+                          <Form.Item name="routing_strategy" className="mb-0">
+                            <Select
+                              allowClear
+                              placeholder="Inherit router default"
+                              style={{ width: "100%" }}
+                              options={[...ROUTING_STRATEGY_OPTIONS]}
+                            />
+                          </Form.Item>
+                        ) : (
+                          <div className="mt-1 p-2 bg-gray-50 rounded-sm">
+                            {routingStrategyLabel(localModelData.model_info?.routing_strategy)}
+                          </div>
+                        )}
+                      </div>
+
                       <div>
                         <Text className="font-medium">
                           Guardrails

ui/litellm-dashboard/src/components/routing_groups/RoutingGroupModal.tsx

@@ -125,7 +125,7 @@ const RoutingGroupModal: React.FC<RoutingGroupModalProps> = ({
               },
             },
           ]}
-          extra="Use this name as the model in API calls — LiteLLM routes the request to one of the group's models."
+          extra="A label for this strategy assignment. The group name is not a callable model: requests still use the member model names, and the strategy picks among each member's deployments."
         >
           <Input placeholder="fast-chat" disabled={mode === "edit"} />
         </Form.Item>

ui/litellm-dashboard/src/components/routing_groups/index.tsx

@@ -1,7 +1,7 @@
 "use client";
 
 import React, { useMemo, useState } from "react";
-import { Button, Card, Flex, Input, Modal, Space, Typography } from "antd";
+import { Alert, Button, Card, Flex, Input, Modal, Space, Typography } from "antd";
 import { PlusOutlined, ReloadOutlined, SearchOutlined } from "@ant-design/icons";
 import { useRoutingGroups, useSaveRoutingGroups } from "@/app/(dashboard)/hooks/routingGroups/useRoutingGroups";
 import { useRouterFields } from "@/app/(dashboard)/hooks/router/useRouterFields";
@@ -101,6 +101,12 @@ const RoutingGroups: React.FC = () => {
 
   return (
     <Space direction="vertical" size={16} className="w-full">
+      <Alert
+        type="warning"
+        showIcon
+        message="Deprecated"
+        description="Routing groups only set the load-balancing strategy for their member model names; the group name is not a callable model. Prefer setting a Routing Strategy directly on the model (Models + Endpoints > Add Model > Advanced Settings). Existing groups keep working, and a strategy set on a model overrides its group."
+      />
       <Card bodyStyle={{ padding: 16 }}>
         <Flex justify="space-between" align="center" gap={12} className="mb-4">
           <Input

ui/litellm-dashboard/src/lib/http/schema.d.ts

@@ -35270,6 +35270,12 @@ export interface components {
             db_model: boolean;
             /** Id */
             id: string | null;
+            /** Routing Strategy */
+            routing_strategy?: string | null;
+            /** Routing Strategy Args */
+            routing_strategy_args?: {
+                [key: string]: unknown;
+            } | null;
             /** Team Id */
             team_id?: string | null;
             /** Team Public Model Name */