From f323b62737f0e8e2719062ea48a8517a2c3a36af Mon Sep 17 00:00:00 2001 From: Souravrajvi0 <144546710+Souravrajvi0@users.noreply.github.com> Date: Wed, 26 Aug 2026 09:39:16 +0000 Subject: [PATCH] fix(router): reject ambiguous raw model fallback across groups Opt-in enable_cross_model_group_collision_check stops a raw litellm_params.model string from load-balancing across unrelated model_name groups. Default stays off for compatibility. --- litellm/router.py | 29 ++++++ .../router_settings_endpoints.py | 8 ++ litellm/types/router.py | 1 + tests/test_litellm/test_router.py | 93 +++++++++++++++++++ 4 files changed, 131 insertions(+) diff --git a/litellm/router.py b/litellm/router.py index 1e9b23b2fa1..38b9d9a9987 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -180,6 +180,7 @@ from litellm.router_utils.router_callbacks.track_deployment_metrics import ( increment_deployment_successes_for_current_minute, ) from litellm.scheduler import FlowItem, Scheduler +from litellm.secret_managers.main import get_secret_bool from litellm.types.llms.openai import ( AllMessageValues, FileTypes, @@ -603,6 +604,7 @@ class Router: health_check_staleness_threshold: int | None = None, health_check_ignore_transient_errors: bool = False, enable_weighted_failover: bool = False, + enable_cross_model_group_collision_check: bool = False, ) -> None: """ Initialize the Router class with the given parameters for caching, reliability, and routing strategy. @@ -639,6 +641,7 @@ class Router: deployment_affinity_ttl_seconds (int): TTL for user-key -> deployment affinity mapping. Defaults to 3600. ignore_invalid_deployments (bool): Ignores invalid deployments, and continues with other deployments. Default is to raise an error. enable_weighted_failover (bool): When True and the routing strategy is "simple-shuffle", a retryable failure on one deployment causes the request to re-pick (weighted) across the other deployments in the same model group before any cross-group fallback runs. Bounded by `max_fallbacks`. Async-only: currently honored by `router.acompletion()` and other async entrypoints. The sync `router.completion()` path falls back to the regular fallback flow. Defaults to False. + enable_cross_model_group_collision_check (bool): When True, refuse the fallback that load-balances a raw `litellm_params.model` string across more than one `model_name` group. Opt-in; also enabled by `LITELLM_ENABLE_CROSS_MODEL_GROUP_COLLISION_CHECK`. Defaults to False. Returns: Router: An instance of the litellm.Router class. @@ -810,6 +813,10 @@ class Router: self.disable_cooldowns = disable_cooldowns self.enable_health_check_routing = enable_health_check_routing self.enable_weighted_failover = enable_weighted_failover + self.enable_cross_model_group_collision_check = bool( + enable_cross_model_group_collision_check + or get_secret_bool("LITELLM_ENABLE_CROSS_MODEL_GROUP_COLLISION_CHECK", False) + ) self.health_check_ignore_transient_errors = health_check_ignore_transient_errors _staleness: Final = health_check_staleness_threshold or ( DEFAULT_HEALTH_CHECK_INTERVAL * DEFAULT_HEALTH_CHECK_STALENESS_MULTIPLIER @@ -10900,6 +10907,7 @@ class Router: "retry_policy", "model_group_alias", "enable_weighted_failover", + "enable_cross_model_group_collision_check", "enable_tag_filtering", "tag_routing_prefix", ] @@ -10938,6 +10946,7 @@ class Router: "model_group_retry_policy", "model_group_alias", "enable_weighted_failover", + "enable_cross_model_group_collision_check", "enable_tag_filtering", "tag_routing_prefix", ] @@ -11332,6 +11341,24 @@ class Router: """ return [m for m in self.model_list if m["litellm_params"]["model"] == model] + def _reject_cross_model_group_collision(self, model: str, deployments: list) -> None: + group_names: Final = frozenset( + str(deployment["model_name"]) + for deployment in deployments + if isinstance(deployment.get("model_name"), str) and deployment["model_name"] + ) + if len(group_names) <= 1: + return + groups: Final = ", ".join(sorted(group_names)) + raise litellm.BadRequestError( + message=( + f"Model '{model}' matches deployments from multiple model groups ({groups}). " + "Request a configured model_name instead of the raw provider model string." + ), + model=model, + llm_provider="", + ) + def _try_early_resolve_deployments_for_model_not_in_names( self, model: str, @@ -11487,6 +11514,8 @@ class Router: request_kwargs=request_kwargs, request_team_id=request_team_id, ) + if self.enable_cross_model_group_collision_check: + self._reject_cross_model_group_collision(model=model, deployments=healthy_deployments) # If the litellm-model lookup produced candidates that access-group # filtering then removed, treat this the same as the by-name path # being emptied: prevent default-model fallback from bypassing the diff --git a/litellm/types/management_endpoints/router_settings_endpoints.py b/litellm/types/management_endpoints/router_settings_endpoints.py index cef180b202a..e3f5c11dc5e 100644 --- a/litellm/types/management_endpoints/router_settings_endpoints.py +++ b/litellm/types/management_endpoints/router_settings_endpoints.py @@ -220,6 +220,14 @@ ROUTER_SETTINGS_FIELDS: Final[list[RouterSettingsField]] = [ field_default=False, ui_field_name="Enable Pre-call Checks", ), + RouterSettingsField( + field_name="enable_cross_model_group_collision_check", + field_type="Boolean", + field_value=None, + field_description=("Reject raw provider model strings that match more than one model_name group"), + field_default=False, + ui_field_name="Cross Model Group Collision Check", + ), RouterSettingsField( field_name="default_litellm_params", field_type="Dictionary", diff --git a/litellm/types/router.py b/litellm/types/router.py index d4c735387a5..3dc4a6c67e3 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -125,6 +125,7 @@ class UpdateRouterConfig(BaseModel): model_group_alias: dict[str, str | dict] | None = {} enable_tag_filtering: bool | None = None tag_routing_prefix: str | None = None + enable_cross_model_group_collision_check: bool | None = None model_config = ConfigDict(protected_namespaces=()) diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index cceb034a20b..f35859c211b 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -10649,3 +10649,96 @@ def test_permission_denied_error_is_retried_when_other_deployments_exist(): ) is True ) + + +def _colliding_raw_model_groups() -> list[dict]: + shared_model = "openai/some-shared-string" + return [ + { + "model_name": "chain-a", + "litellm_params": { + "model": shared_model, + "api_key": "key-a", + "mock_response": "from-a", + }, + }, + { + "model_name": "chain-b", + "litellm_params": { + "model": shared_model, + "api_key": "key-b", + "mock_response": "from-b", + }, + }, + ] + + +def test_raw_litellm_model_fallback_spans_unrelated_groups_by_default(): + """Issue #38216: a raw litellm_params.model string load-balances across every group that shares it.""" + router = litellm.Router(model_list=_colliding_raw_model_groups()) + + _model, deployments = router._common_checks_available_deployment( + model="openai/some-shared-string" + ) + + assert {d["model_name"] for d in deployments} == {"chain-a", "chain-b"} + assert router.get_available_deployment(model="openai/some-shared-string") is not None + + +def test_cross_model_group_collision_check_rejects_ambiguous_raw_model(): + router = litellm.Router( + model_list=_colliding_raw_model_groups(), + enable_cross_model_group_collision_check=True, + ) + + with pytest.raises(litellm.BadRequestError, match="multiple model groups"): + router.get_available_deployment(model="openai/some-shared-string") + + deployment = router.get_available_deployment(model="chain-a") + assert deployment["model_name"] == "chain-a" + + +def test_cross_model_group_collision_check_allows_same_group_and_unambiguous_raw_model(): + router = litellm.Router( + model_list=[ + { + "model_name": "chain-a", + "litellm_params": { + "model": "openai/some-shared-string", + "api_key": "key-a", + "mock_response": "from-a-1", + }, + }, + { + "model_name": "chain-a", + "litellm_params": { + "model": "openai/some-shared-string", + "api_key": "key-a-2", + "mock_response": "from-a-2", + }, + }, + { + "model_name": "chain-b", + "litellm_params": { + "model": "openai/other-string", + "api_key": "key-b", + "mock_response": "from-b", + }, + }, + ], + enable_cross_model_group_collision_check=True, + ) + + _model, deployments = router._common_checks_available_deployment( + model="openai/some-shared-string" + ) + assert {d["model_name"] for d in deployments} == {"chain-a"} + assert len(deployments) == 2 + + +def test_cross_model_group_collision_check_env_enables_guard(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("LITELLM_ENABLE_CROSS_MODEL_GROUP_COLLISION_CHECK", "true") + router = litellm.Router(model_list=_colliding_raw_model_groups()) + + with pytest.raises(litellm.BadRequestError, match="multiple model groups"): + router.get_available_deployment(model="openai/some-shared-string")