fix(router): resolve model_group_alias before pre-routing strategy dispatch (#38382)

A model_group_alias whose target is an auto-router shows up in /v1/models and
/model_group/info but 400s on call with "Unmapped LLM provider for this
endpoint. You passed model=complexity_router, custom_llm_provider=auto_router".
async_pre_routing_hook picks the strategy using the name the caller passed,
while the alias is only resolved further down in
_common_checks_available_deployment, so the four strategy registries, all keyed
by the marker deployment's model_name, never match. The hook then declines, and
the auto_router/ marker deployment goes out as if it were a real model

Resolve the alias once at the top of the hook, for lookups only, so the
registries, the tag-filtering escape hatch and the marker's forwardable params
all see the name they are keyed under. The caller-facing name is untouched:
spend metadata is stamped before routing and the response still carries the
alias the client sent

Second half, so the same symptom cannot reach a provider through the entry
points this does not fix (the sync selection path that never runs the hook, a
team-scoped router keyed on its internal name), a group that resolves only to
strategy markers is no longer callable: it raises a BadRequestError naming the
marker instead of handing the auto_router/ pseudo-model to the provider
This commit is contained in:
tin-berri 2026-08-26 13:38:55 -07:00 • committed by GitHub
parent f6571a653f
commit e4037978f1
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 145 additions and 9 deletions

View file

@ -205,6 +205,7 @@ from litellm.types.router import (
PreRoutingStrategy,
RetryPolicy,
RouterCacheEnum,
RouterErrors,
RouterGeneralSettings,
RouterModelGroupAliasItem,
RouterRateLimitError,
@ -11520,10 +11521,8 @@ class Router:
# If still no deployments after checking for fallbacks, raise an error
if len(healthy_deployments) == 0:
message: Final = f"You passed in model={model}. There are no healthy deployments for this model"
raise litellm.BadRequestError(
message=message,
message=f"You passed in model={model}. {RouterErrors.no_healthy_deployments.value}",
model=model,
llm_provider="",
)
@ -11534,11 +11533,18 @@ class Router:
] # update the model to the actual value if an alias has been passed in
marker_flags: Final = tuple(self._is_strategy_marker_deployment(d) for d in healthy_deployments)
if all(marker_flags) or not any(marker_flags):
if not any(marker_flags):
return model, healthy_deployments
return model, [ # mutable-ok: matches this function's list contract expected by downstream filters
selectable: Final = [ # mutable-ok: matches this function's list contract expected by downstream filters
d for d, is_marker in zip(healthy_deployments, marker_flags, strict=True) if not is_marker
]
if not selectable:
raise litellm.BadRequestError(
message=f"You passed in model={model}. {RouterErrors.only_strategy_marker_deployments.value}",
model=model,
llm_provider="",
)
return model, selectable
def _filter_deployments_by_model_access_groups(
self,
@ -12127,7 +12133,15 @@ class Router:
This hook is called before the routing decision is made.
Used for the litellm auto-router to modify the request before the routing decision is made.
`model` is whatever the caller asked for, which may be a `model_group_alias` key, while the
strategy registries and the marker deployment are keyed by the marker's own `model_name`, so
every lookup below resolves the alias first. Only the lookups: the caller-facing name stays
the alias, since spend metadata is stamped before routing and the response carries the tier
group the strategy picked.
"""
registered_model_name: Final = self._get_model_from_alias(model=model) or model
#########################################################
# Run the routing-plugin pipeline, if any plugins are configured.
# Plugins narrow the candidate deployment pool (consumed later by
@ -12135,9 +12149,13 @@ class Router:
# downstream strategies (auto-router, complexity-router, ...) to read.
#########################################################
if self.routing_plugins:
await self._run_routing_plugins(model=model, request_kwargs=request_kwargs, messages=messages)
await self._run_routing_plugins(
model=registered_model_name, request_kwargs=request_kwargs, messages=messages
)
selected_strategy: Final = self._select_pre_routing_strategy(model=model, request_kwargs=request_kwargs)
selected_strategy: Final = self._select_pre_routing_strategy(
model=registered_model_name, request_kwargs=request_kwargs
)
if selected_strategy is None:
self._record_routing_decision(request_kwargs=request_kwargs, routing_decision=None)
self._stamp_or_clear_metadata_key(
@ -12152,7 +12170,7 @@ class Router:
return None
pre_routing_hook_response: Final = await selected_strategy.strategy.async_pre_routing_hook(
model=model,
model=registered_model_name,
request_kwargs=request_kwargs,
messages=messages,
input=input,
@ -12204,7 +12222,7 @@ class Router:
# Per-tier `litellm_params` on the hook response are deliberate overrides
# the caller applies on top, so those keys are never forwarded here.
marker_params: Final = (
self._forwardable_alias_marker_params(model=model, strategy_tags=selected_strategy.tags)
self._forwardable_alias_marker_params(model=registered_model_name, strategy_tags=selected_strategy.tags)
if pre_routing_hook_response is not None
else ()
)

View file

@ -576,6 +576,11 @@ class RouterErrors(enum.Enum):
no_deployments_available = "No deployments available for selected model"
no_deployments_with_tag_routing = "Not allowed to access model due to tags configuration"
no_deployments_with_provider_budget_routing = "No deployments available - crossed budget"
no_healthy_deployments = "There are no healthy deployments for this model"
only_strategy_marker_deployments = (
"Every deployment for it is a strategy router marker (auto_router/...), which is not a callable "
"model, and no pre-routing strategy selected a deployment for this request"
)
class AllowedFailsPolicy(BaseModel):

View file

@ -8855,6 +8855,119 @@ class TestAutoRoutedRequestMarker:
assert AUTO_ROUTED_REQUEST_METADATA_KEY not in request_kwargs["metadata"]
class TestModelGroupAliasReachesPreRoutingStrategies:
"""A `model_group_alias` whose target is a strategy router must dispatch exactly like the
router's own model_name. The four strategy registries are keyed by the marker deployment's
model_name, so the alias has to be resolved before the pre-routing hook looks anything up,
and a group that resolves only to markers is not callable at all (LIT-4664)."""
MARKER_TIMEOUT = 42.0
REGISTRY_NAMES = ("auto_routers", "complexity_routers", "adaptive_routers", "quality_routers")
class _RewriteStrategy:
async def async_pre_routing_hook(
self, model, request_kwargs, messages=None, input=None, specific_deployment=False
):
from litellm.types.router import PreRoutingHookResponse
return PreRoutingHookResponse(model="gemini-flash", messages=messages)
@classmethod
def _router(cls, registry_name: str | None) -> "litellm.Router":
from litellm.types.router import TaggedPreRoutingStrategy
tiers = dict.fromkeys(("SIMPLE", "MEDIUM", "COMPLEX", "REASONING"), "gemini-flash")
router = litellm.Router(
model_list=[
{
"model_name": "smart-route",
"litellm_params": {
"model": "auto_router/complexity_router",
"complexity_router_config": {"tiers": tiers},
"complexity_router_default_model": "gemini-flash",
"timeout": cls.MARKER_TIMEOUT,
},
},
{
"model_name": "gemini-flash",
"litellm_params": {"model": "gemini/gemini-3.6-flash", "mock_response": "routed by the tier"},
},
],
model_group_alias={"smart-alias": "smart-route"},
)
for name in cls.REGISTRY_NAMES:
setattr(router, name, {})
if registry_name is not None:
setattr(
router,
registry_name,
{"smart-route": [TaggedPreRoutingStrategy(tags=(), strategy=cls._RewriteStrategy())]},
)
return router
@staticmethod
def _messages() -> list[dict[str, str]]:
return [{"role": "user", "content": "What is the capital of France?"}]
@pytest.mark.parametrize("registry_name", REGISTRY_NAMES)
@pytest.mark.asyncio
async def test_alias_dispatches_to_the_strategy_registered_under_the_target(self, registry_name):
from litellm.constants import AUTO_ROUTED_REQUEST_METADATA_KEY
router = self._router(registry_name)
request_kwargs = {"metadata": {}}
response = await router.async_pre_routing_hook(
model="smart-alias", request_kwargs=request_kwargs, messages=self._messages()
)
assert response is not None
assert response.model == "gemini-flash"
assert request_kwargs["metadata"][AUTO_ROUTED_REQUEST_METADATA_KEY] is True
@pytest.mark.asyncio
async def test_alias_call_still_forwards_the_marker_own_params_to_the_routed_tier(self):
router = self._router("auto_routers")
request_kwargs = {"metadata": {}}
await router.async_pre_routing_hook(
model="smart-alias", request_kwargs=request_kwargs, messages=self._messages()
)
assert request_kwargs["timeout"] == self.MARKER_TIMEOUT
@pytest.mark.asyncio
async def test_alias_deployment_selection_lands_on_the_tier_never_the_marker(self):
router = self._router("auto_routers")
deployment = await router.async_get_available_deployment(
model="smart-alias", request_kwargs={"metadata": {}}, messages=self._messages()
)
assert deployment["litellm_params"]["model"] == "gemini/gemini-3.6-flash"
@pytest.mark.asyncio
async def test_alias_call_completes_and_still_bills_the_name_the_caller_sent(self):
router = self._router("auto_routers")
metadata: dict = {}
response = await router.acompletion(
model="smart-alias", messages=self._messages(), metadata=metadata
)
assert response.choices[0].message.content == "routed by the tier"
assert metadata["model_group"] == "smart-alias"
assert metadata["model_group_alias"] == "smart-alias"
def test_a_group_of_only_markers_is_not_a_callable_model(self):
router = self._router(None)
with pytest.raises(litellm.BadRequestError, match="strategy router marker"):
router.get_available_deployment(
model="smart-route", messages=self._messages(), request_kwargs={"metadata": {}}
)
@pytest.mark.usefixtures("local_model_cost_map")
class TestAzureBaseModelFallbackLogging:
"""When an azure deployment has no base_model but its model name is a known