From e4037978f1ed3038fc8cc6563e5a6214bf369ce0 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Wed, 26 Aug 2026 13:38:55 -0700 Subject: [PATCH] fix(router): resolve model_group_alias before pre-routing strategy dispatch (#38382) A model_group_alias whose target is an auto-router shows up in /v1/models and /model_group/info but 400s on call with "Unmapped LLM provider for this endpoint. You passed model=complexity_router, custom_llm_provider=auto_router". async_pre_routing_hook picks the strategy using the name the caller passed, while the alias is only resolved further down in _common_checks_available_deployment, so the four strategy registries, all keyed by the marker deployment's model_name, never match. The hook then declines, and the auto_router/ marker deployment goes out as if it were a real model Resolve the alias once at the top of the hook, for lookups only, so the registries, the tag-filtering escape hatch and the marker's forwardable params all see the name they are keyed under. The caller-facing name is untouched: spend metadata is stamped before routing and the response still carries the alias the client sent Second half, so the same symptom cannot reach a provider through the entry points this does not fix (the sync selection path that never runs the hook, a team-scoped router keyed on its internal name), a group that resolves only to strategy markers is no longer callable: it raises a BadRequestError naming the marker instead of handing the auto_router/ pseudo-model to the provider --- litellm/router.py | 36 +++++++--- litellm/types/router.py | 5 ++ tests/test_litellm/test_router.py | 113 ++++++++++++++++++++++++++++++ 3 files changed, 145 insertions(+), 9 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index fc9743ff43d..2dc79670c07 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -205,6 +205,7 @@ from litellm.types.router import ( PreRoutingStrategy, RetryPolicy, RouterCacheEnum, + RouterErrors, RouterGeneralSettings, RouterModelGroupAliasItem, RouterRateLimitError, @@ -11520,10 +11521,8 @@ class Router: # If still no deployments after checking for fallbacks, raise an error if len(healthy_deployments) == 0: - message: Final = f"You passed in model={model}. There are no healthy deployments for this model" - raise litellm.BadRequestError( - message=message, + message=f"You passed in model={model}. {RouterErrors.no_healthy_deployments.value}", model=model, llm_provider="", ) @@ -11534,11 +11533,18 @@ class Router: ] # update the model to the actual value if an alias has been passed in marker_flags: Final = tuple(self._is_strategy_marker_deployment(d) for d in healthy_deployments) - if all(marker_flags) or not any(marker_flags): + if not any(marker_flags): return model, healthy_deployments - return model, [ # mutable-ok: matches this function's list contract expected by downstream filters + selectable: Final = [ # mutable-ok: matches this function's list contract expected by downstream filters d for d, is_marker in zip(healthy_deployments, marker_flags, strict=True) if not is_marker ] + if not selectable: + raise litellm.BadRequestError( + message=f"You passed in model={model}. {RouterErrors.only_strategy_marker_deployments.value}", + model=model, + llm_provider="", + ) + return model, selectable def _filter_deployments_by_model_access_groups( self, @@ -12127,7 +12133,15 @@ class Router: This hook is called before the routing decision is made. Used for the litellm auto-router to modify the request before the routing decision is made. + + `model` is whatever the caller asked for, which may be a `model_group_alias` key, while the + strategy registries and the marker deployment are keyed by the marker's own `model_name`, so + every lookup below resolves the alias first. Only the lookups: the caller-facing name stays + the alias, since spend metadata is stamped before routing and the response carries the tier + group the strategy picked. """ + registered_model_name: Final = self._get_model_from_alias(model=model) or model + ######################################################### # Run the routing-plugin pipeline, if any plugins are configured. # Plugins narrow the candidate deployment pool (consumed later by @@ -12135,9 +12149,13 @@ class Router: # downstream strategies (auto-router, complexity-router, ...) to read. ######################################################### if self.routing_plugins: - await self._run_routing_plugins(model=model, request_kwargs=request_kwargs, messages=messages) + await self._run_routing_plugins( + model=registered_model_name, request_kwargs=request_kwargs, messages=messages + ) - selected_strategy: Final = self._select_pre_routing_strategy(model=model, request_kwargs=request_kwargs) + selected_strategy: Final = self._select_pre_routing_strategy( + model=registered_model_name, request_kwargs=request_kwargs + ) if selected_strategy is None: self._record_routing_decision(request_kwargs=request_kwargs, routing_decision=None) self._stamp_or_clear_metadata_key( @@ -12152,7 +12170,7 @@ class Router: return None pre_routing_hook_response: Final = await selected_strategy.strategy.async_pre_routing_hook( - model=model, + model=registered_model_name, request_kwargs=request_kwargs, messages=messages, input=input, @@ -12204,7 +12222,7 @@ class Router: # Per-tier `litellm_params` on the hook response are deliberate overrides # the caller applies on top, so those keys are never forwarded here. marker_params: Final = ( - self._forwardable_alias_marker_params(model=model, strategy_tags=selected_strategy.tags) + self._forwardable_alias_marker_params(model=registered_model_name, strategy_tags=selected_strategy.tags) if pre_routing_hook_response is not None else () ) diff --git a/litellm/types/router.py b/litellm/types/router.py index d4c735387a5..a3335be2b2b 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -576,6 +576,11 @@ class RouterErrors(enum.Enum): no_deployments_available = "No deployments available for selected model" no_deployments_with_tag_routing = "Not allowed to access model due to tags configuration" no_deployments_with_provider_budget_routing = "No deployments available - crossed budget" + no_healthy_deployments = "There are no healthy deployments for this model" + only_strategy_marker_deployments = ( + "Every deployment for it is a strategy router marker (auto_router/...), which is not a callable " + "model, and no pre-routing strategy selected a deployment for this request" + ) class AllowedFailsPolicy(BaseModel): diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index cceb034a20b..2e81d5f9f46 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -8855,6 +8855,119 @@ class TestAutoRoutedRequestMarker: assert AUTO_ROUTED_REQUEST_METADATA_KEY not in request_kwargs["metadata"] +class TestModelGroupAliasReachesPreRoutingStrategies: + """A `model_group_alias` whose target is a strategy router must dispatch exactly like the + router's own model_name. The four strategy registries are keyed by the marker deployment's + model_name, so the alias has to be resolved before the pre-routing hook looks anything up, + and a group that resolves only to markers is not callable at all (LIT-4664).""" + + MARKER_TIMEOUT = 42.0 + REGISTRY_NAMES = ("auto_routers", "complexity_routers", "adaptive_routers", "quality_routers") + + class _RewriteStrategy: + async def async_pre_routing_hook( + self, model, request_kwargs, messages=None, input=None, specific_deployment=False + ): + from litellm.types.router import PreRoutingHookResponse + + return PreRoutingHookResponse(model="gemini-flash", messages=messages) + + @classmethod + def _router(cls, registry_name: str | None) -> "litellm.Router": + from litellm.types.router import TaggedPreRoutingStrategy + + tiers = dict.fromkeys(("SIMPLE", "MEDIUM", "COMPLEX", "REASONING"), "gemini-flash") + router = litellm.Router( + model_list=[ + { + "model_name": "smart-route", + "litellm_params": { + "model": "auto_router/complexity_router", + "complexity_router_config": {"tiers": tiers}, + "complexity_router_default_model": "gemini-flash", + "timeout": cls.MARKER_TIMEOUT, + }, + }, + { + "model_name": "gemini-flash", + "litellm_params": {"model": "gemini/gemini-3.6-flash", "mock_response": "routed by the tier"}, + }, + ], + model_group_alias={"smart-alias": "smart-route"}, + ) + for name in cls.REGISTRY_NAMES: + setattr(router, name, {}) + if registry_name is not None: + setattr( + router, + registry_name, + {"smart-route": [TaggedPreRoutingStrategy(tags=(), strategy=cls._RewriteStrategy())]}, + ) + return router + + @staticmethod + def _messages() -> list[dict[str, str]]: + return [{"role": "user", "content": "What is the capital of France?"}] + + @pytest.mark.parametrize("registry_name", REGISTRY_NAMES) + @pytest.mark.asyncio + async def test_alias_dispatches_to_the_strategy_registered_under_the_target(self, registry_name): + from litellm.constants import AUTO_ROUTED_REQUEST_METADATA_KEY + + router = self._router(registry_name) + request_kwargs = {"metadata": {}} + + response = await router.async_pre_routing_hook( + model="smart-alias", request_kwargs=request_kwargs, messages=self._messages() + ) + + assert response is not None + assert response.model == "gemini-flash" + assert request_kwargs["metadata"][AUTO_ROUTED_REQUEST_METADATA_KEY] is True + + @pytest.mark.asyncio + async def test_alias_call_still_forwards_the_marker_own_params_to_the_routed_tier(self): + router = self._router("auto_routers") + request_kwargs = {"metadata": {}} + + await router.async_pre_routing_hook( + model="smart-alias", request_kwargs=request_kwargs, messages=self._messages() + ) + + assert request_kwargs["timeout"] == self.MARKER_TIMEOUT + + @pytest.mark.asyncio + async def test_alias_deployment_selection_lands_on_the_tier_never_the_marker(self): + router = self._router("auto_routers") + + deployment = await router.async_get_available_deployment( + model="smart-alias", request_kwargs={"metadata": {}}, messages=self._messages() + ) + + assert deployment["litellm_params"]["model"] == "gemini/gemini-3.6-flash" + + @pytest.mark.asyncio + async def test_alias_call_completes_and_still_bills_the_name_the_caller_sent(self): + router = self._router("auto_routers") + metadata: dict = {} + + response = await router.acompletion( + model="smart-alias", messages=self._messages(), metadata=metadata + ) + + assert response.choices[0].message.content == "routed by the tier" + assert metadata["model_group"] == "smart-alias" + assert metadata["model_group_alias"] == "smart-alias" + + def test_a_group_of_only_markers_is_not_a_callable_model(self): + router = self._router(None) + + with pytest.raises(litellm.BadRequestError, match="strategy router marker"): + router.get_available_deployment( + model="smart-route", messages=self._messages(), request_kwargs={"metadata": {}} + ) + + @pytest.mark.usefixtures("local_model_cost_map") class TestAzureBaseModelFallbackLogging: """When an azure deployment has no base_model but its model name is a known