fix(router): fall back on invalid routing_strategy_args and evict orphaned model-group selectors

litellm/router.py

@@ -1169,6 +1169,60 @@ class Router:
             )
         return valid[0] if valid else None
 
+    def _build_model_group_selector(self, strategy: str, args: Mapping[str, object]) -> object | None:
+        try:
+            return self._build_strategy_selector(strategy=strategy, routing_strategy_args=args)
+        except (TypeError, ValueError):
+            return None
+
+    def _live_model_group_selector_keys(self) -> frozenset[str]:
+        """
+        Composite selector-cache keys still referenced by some model group's
+        `model_info` strategy config. Used to evict selectors orphaned by model
+        edits or deletions before caching a newly built one.
+        """
+        return frozenset(
+            f"{strategy}|{json.dumps(args, sort_keys=True, default=str)}"
+            for name in tuple(self.model_name_to_deployment_indices)
+            if (config := self._get_model_group_strategy_config(name)) is not None
+            for strategy, args in (config,)
+            if args and strategy != "simple-shuffle"
+        )
+
+    def _resolve_model_group_context(
+        self, model: str, strategy: str, args: Mapping[str, object]
+    ) -> tuple[str, object | None] | None:
+        """
+        Selector resolution for a `model_info.routing_strategy` hit. Returns
+        None when the selector cannot be built (invalid `routing_strategy_args`),
+        so the caller falls through to the legacy routing-group / top-level
+        strategy instead of failing the request; the failure is reported once
+        per model_name.
+        """
+        if strategy == "simple-shuffle" or not args:
+            verbose_router_logger.debug("routing_group=model-info model=%s strategy=%s", model, strategy)
+            return strategy, self._get_override_strategy_selector(strategy)
+        selector_key: Final = f"{strategy}|{json.dumps(args, sort_keys=True, default=str)}"
+        with self._override_selectors_lock:
+            if selector_key not in self._override_selectors:
+                built: Final = self._build_model_group_selector(strategy, args)
+                if built is None:
+                    if model not in self._warned_model_group_strategy_models:
+                        self._warned_model_group_strategy_models.add(model)
+                        verbose_router_logger.warning(
+                            "model_info.routing_strategy_args for model_group '%s' cannot initialize strategy "
+                            "'%s'; falling back to the routing-group / top-level strategy.",
+                            model,
+                            strategy,
+                        )
+                    return None
+                live_keys: Final = self._live_model_group_selector_keys()
+                stale: Final = tuple(k for k in self._override_selectors if "|" in k and k not in live_keys)
+                self._unregister_router_selectors(tuple(self._override_selectors.pop(k) for k in stale))
+                self._override_selectors[selector_key] = built
+            verbose_router_logger.debug("routing_group=model-info model=%s strategy=%s", model, strategy)
+            return strategy, self._override_selectors[selector_key]
+
     def _get_routing_context(self, model: str, request_kwargs: dict | None = None) -> tuple[str | None, Any | None]:
         """
         Resolves the routing strategy and selector to use for the given model.
@@ -1191,19 +1245,11 @@ class Router:
             return override, self._get_override_strategy_selector(override)
 
         model_group_config: Final = self._get_model_group_strategy_config(model)
-        if model_group_config is not None:
-            mg_strategy, mg_args = model_group_config
-            verbose_router_logger.debug("routing_group=model-info model=%s strategy=%s", model, mg_strategy)
-            if not mg_args:
-                return mg_strategy, self._get_override_strategy_selector(mg_strategy)
-            selector_key: Final = f"{mg_strategy}|{json.dumps(mg_args, sort_keys=True, default=str)}"
-            with self._override_selectors_lock:
-                if selector_key not in self._override_selectors:
-                    self._override_selectors[selector_key] = self._build_strategy_selector(
-                        strategy=mg_strategy,
-                        routing_strategy_args=mg_args,
-                    )
-                return mg_strategy, self._override_selectors[selector_key]
+        mg_resolution: Final = (
+            self._resolve_model_group_context(model, *model_group_config) if model_group_config is not None else None
+        )
+        if mg_resolution is not None:
+            return mg_resolution
 
         group_name: Final = self._model_to_group.get(model)
         if group_name is None:

tests/test_litellm/router_strategy/test_model_group_routing_strategy.py

@@ -138,6 +138,78 @@ def test_invalid_value_ignored_with_warning(caplog):
     assert any("unsupported value" in r.getMessage() for r in caplog.records)
 
 
+def test_invalid_args_fall_back_without_failing_traffic(caplog):
+    router = _build_router(
+        [
+            _deployment(
+                "quality",
+                "openai/gpt-4o",
+                "d1",
+                {"routing_strategy": "latency-based-routing", "routing_strategy_args": {"ttl": "bogus"}},
+            )
+        ],
+        routing_strategy="cost-based-routing",
+    )
+    with caplog.at_level(logging.WARNING, logger="LiteLLM Router"):
+        strategy, selector = router._get_routing_context("quality")
+        router._get_routing_context("quality")
+    assert strategy == "cost-based-routing"
+    assert selector is router.lowestcost_logger
+    warnings = [r for r in caplog.records if "cannot initialize strategy" in r.getMessage()]
+    assert len(warnings) == 1
+
+
+def test_simple_shuffle_with_args_keeps_shuffle_semantics(caplog):
+    router = _build_router(
+        [
+            _deployment(
+                "quality",
+                "openai/gpt-4o",
+                "d1",
+                {"routing_strategy": "simple-shuffle", "routing_strategy_args": {"ignored": 1}},
+            )
+        ],
+        routing_strategy="latency-based-routing",
+    )
+    with caplog.at_level(logging.WARNING, logger="LiteLLM Router"):
+        assert router._get_routing_context("quality") == ("simple-shuffle", None)
+    assert not any("cannot initialize strategy" in r.getMessage() for r in caplog.records)
+
+
+def test_stale_selector_evicted_when_args_change():
+    router = _build_router(
+        [
+            _deployment(
+                "quality",
+                "openai/gpt-4o",
+                "d1",
+                {"routing_strategy": "latency-based-routing", "routing_strategy_args": {"ttl": 120}},
+            ),
+            _deployment(
+                "other",
+                "openai/gpt-4o-mini",
+                "d2",
+                {"routing_strategy": "latency-based-routing", "routing_strategy_args": {"ttl": 600}},
+            ),
+        ]
+    )
+    _, old_selector = router._get_routing_context("quality")
+    _, kept_selector = router._get_routing_context("other")
+    old_keys = {k for k in router._override_selectors if "|" in k}
+    assert len(old_keys) == 2
+
+    for idx in router.model_name_to_deployment_indices["quality"]:
+        router.model_list[idx]["model_info"]["routing_strategy_args"] = {"ttl": 240}
+
+    _, new_selector = router._get_routing_context("quality")
+    assert new_selector is not old_selector
+    assert new_selector.routing_args.ttl == 240
+    remaining_keys = {k for k in router._override_selectors if "|" in k}
+    assert len(remaining_keys) == 2
+    assert all(c is not old_selector for c in litellm.callbacks)
+    assert router._get_routing_context("other")[1] is kept_selector
+
+
 def test_selector_shared_across_model_groups_with_identical_config():
     router = _build_router(
         [