mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-12 08:42:18 +00:00
fix(router): resolve model_group_alias before pre-routing strategy dispatch (#38382)
A model_group_alias whose target is an auto-router shows up in /v1/models and /model_group/info but 400s on call with "Unmapped LLM provider for this endpoint. You passed model=complexity_router, custom_llm_provider=auto_router". async_pre_routing_hook picks the strategy using the name the caller passed, while the alias is only resolved further down in _common_checks_available_deployment, so the four strategy registries, all keyed by the marker deployment's model_name, never match. The hook then declines, and the auto_router/ marker deployment goes out as if it were a real model Resolve the alias once at the top of the hook, for lookups only, so the registries, the tag-filtering escape hatch and the marker's forwardable params all see the name they are keyed under. The caller-facing name is untouched: spend metadata is stamped before routing and the response still carries the alias the client sent Second half, so the same symptom cannot reach a provider through the entry points this does not fix (the sync selection path that never runs the hook, a team-scoped router keyed on its internal name), a group that resolves only to strategy markers is no longer callable: it raises a BadRequestError naming the marker instead of handing the auto_router/ pseudo-model to the provider
This commit is contained in:
parent
f6571a653f
commit
e4037978f1
3 changed files with 145 additions and 9 deletions
|
|
@ -205,6 +205,7 @@ from litellm.types.router import (
|
|||
PreRoutingStrategy,
|
||||
RetryPolicy,
|
||||
RouterCacheEnum,
|
||||
RouterErrors,
|
||||
RouterGeneralSettings,
|
||||
RouterModelGroupAliasItem,
|
||||
RouterRateLimitError,
|
||||
|
|
@ -11520,10 +11521,8 @@ class Router:
|
|||
|
||||
# If still no deployments after checking for fallbacks, raise an error
|
||||
if len(healthy_deployments) == 0:
|
||||
message: Final = f"You passed in model={model}. There are no healthy deployments for this model"
|
||||
|
||||
raise litellm.BadRequestError(
|
||||
message=message,
|
||||
message=f"You passed in model={model}. {RouterErrors.no_healthy_deployments.value}",
|
||||
model=model,
|
||||
llm_provider="",
|
||||
)
|
||||
|
|
@ -11534,11 +11533,18 @@ class Router:
|
|||
] # update the model to the actual value if an alias has been passed in
|
||||
|
||||
marker_flags: Final = tuple(self._is_strategy_marker_deployment(d) for d in healthy_deployments)
|
||||
if all(marker_flags) or not any(marker_flags):
|
||||
if not any(marker_flags):
|
||||
return model, healthy_deployments
|
||||
return model, [ # mutable-ok: matches this function's list contract expected by downstream filters
|
||||
selectable: Final = [ # mutable-ok: matches this function's list contract expected by downstream filters
|
||||
d for d, is_marker in zip(healthy_deployments, marker_flags, strict=True) if not is_marker
|
||||
]
|
||||
if not selectable:
|
||||
raise litellm.BadRequestError(
|
||||
message=f"You passed in model={model}. {RouterErrors.only_strategy_marker_deployments.value}",
|
||||
model=model,
|
||||
llm_provider="",
|
||||
)
|
||||
return model, selectable
|
||||
|
||||
def _filter_deployments_by_model_access_groups(
|
||||
self,
|
||||
|
|
@ -12127,7 +12133,15 @@ class Router:
|
|||
This hook is called before the routing decision is made.
|
||||
|
||||
Used for the litellm auto-router to modify the request before the routing decision is made.
|
||||
|
||||
`model` is whatever the caller asked for, which may be a `model_group_alias` key, while the
|
||||
strategy registries and the marker deployment are keyed by the marker's own `model_name`, so
|
||||
every lookup below resolves the alias first. Only the lookups: the caller-facing name stays
|
||||
the alias, since spend metadata is stamped before routing and the response carries the tier
|
||||
group the strategy picked.
|
||||
"""
|
||||
registered_model_name: Final = self._get_model_from_alias(model=model) or model
|
||||
|
||||
#########################################################
|
||||
# Run the routing-plugin pipeline, if any plugins are configured.
|
||||
# Plugins narrow the candidate deployment pool (consumed later by
|
||||
|
|
@ -12135,9 +12149,13 @@ class Router:
|
|||
# downstream strategies (auto-router, complexity-router, ...) to read.
|
||||
#########################################################
|
||||
if self.routing_plugins:
|
||||
await self._run_routing_plugins(model=model, request_kwargs=request_kwargs, messages=messages)
|
||||
await self._run_routing_plugins(
|
||||
model=registered_model_name, request_kwargs=request_kwargs, messages=messages
|
||||
)
|
||||
|
||||
selected_strategy: Final = self._select_pre_routing_strategy(model=model, request_kwargs=request_kwargs)
|
||||
selected_strategy: Final = self._select_pre_routing_strategy(
|
||||
model=registered_model_name, request_kwargs=request_kwargs
|
||||
)
|
||||
if selected_strategy is None:
|
||||
self._record_routing_decision(request_kwargs=request_kwargs, routing_decision=None)
|
||||
self._stamp_or_clear_metadata_key(
|
||||
|
|
@ -12152,7 +12170,7 @@ class Router:
|
|||
return None
|
||||
|
||||
pre_routing_hook_response: Final = await selected_strategy.strategy.async_pre_routing_hook(
|
||||
model=model,
|
||||
model=registered_model_name,
|
||||
request_kwargs=request_kwargs,
|
||||
messages=messages,
|
||||
input=input,
|
||||
|
|
@ -12204,7 +12222,7 @@ class Router:
|
|||
# Per-tier `litellm_params` on the hook response are deliberate overrides
|
||||
# the caller applies on top, so those keys are never forwarded here.
|
||||
marker_params: Final = (
|
||||
self._forwardable_alias_marker_params(model=model, strategy_tags=selected_strategy.tags)
|
||||
self._forwardable_alias_marker_params(model=registered_model_name, strategy_tags=selected_strategy.tags)
|
||||
if pre_routing_hook_response is not None
|
||||
else ()
|
||||
)
|
||||
|
|
|
|||
|
|
@ -576,6 +576,11 @@ class RouterErrors(enum.Enum):
|
|||
no_deployments_available = "No deployments available for selected model"
|
||||
no_deployments_with_tag_routing = "Not allowed to access model due to tags configuration"
|
||||
no_deployments_with_provider_budget_routing = "No deployments available - crossed budget"
|
||||
no_healthy_deployments = "There are no healthy deployments for this model"
|
||||
only_strategy_marker_deployments = (
|
||||
"Every deployment for it is a strategy router marker (auto_router/...), which is not a callable "
|
||||
"model, and no pre-routing strategy selected a deployment for this request"
|
||||
)
|
||||
|
||||
|
||||
class AllowedFailsPolicy(BaseModel):
|
||||
|
|
|
|||
|
|
@ -8855,6 +8855,119 @@ class TestAutoRoutedRequestMarker:
|
|||
assert AUTO_ROUTED_REQUEST_METADATA_KEY not in request_kwargs["metadata"]
|
||||
|
||||
|
||||
class TestModelGroupAliasReachesPreRoutingStrategies:
|
||||
"""A `model_group_alias` whose target is a strategy router must dispatch exactly like the
|
||||
router's own model_name. The four strategy registries are keyed by the marker deployment's
|
||||
model_name, so the alias has to be resolved before the pre-routing hook looks anything up,
|
||||
and a group that resolves only to markers is not callable at all (LIT-4664)."""
|
||||
|
||||
MARKER_TIMEOUT = 42.0
|
||||
REGISTRY_NAMES = ("auto_routers", "complexity_routers", "adaptive_routers", "quality_routers")
|
||||
|
||||
class _RewriteStrategy:
|
||||
async def async_pre_routing_hook(
|
||||
self, model, request_kwargs, messages=None, input=None, specific_deployment=False
|
||||
):
|
||||
from litellm.types.router import PreRoutingHookResponse
|
||||
|
||||
return PreRoutingHookResponse(model="gemini-flash", messages=messages)
|
||||
|
||||
@classmethod
|
||||
def _router(cls, registry_name: str | None) -> "litellm.Router":
|
||||
from litellm.types.router import TaggedPreRoutingStrategy
|
||||
|
||||
tiers = dict.fromkeys(("SIMPLE", "MEDIUM", "COMPLEX", "REASONING"), "gemini-flash")
|
||||
router = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "smart-route",
|
||||
"litellm_params": {
|
||||
"model": "auto_router/complexity_router",
|
||||
"complexity_router_config": {"tiers": tiers},
|
||||
"complexity_router_default_model": "gemini-flash",
|
||||
"timeout": cls.MARKER_TIMEOUT,
|
||||
},
|
||||
},
|
||||
{
|
||||
"model_name": "gemini-flash",
|
||||
"litellm_params": {"model": "gemini/gemini-3.6-flash", "mock_response": "routed by the tier"},
|
||||
},
|
||||
],
|
||||
model_group_alias={"smart-alias": "smart-route"},
|
||||
)
|
||||
for name in cls.REGISTRY_NAMES:
|
||||
setattr(router, name, {})
|
||||
if registry_name is not None:
|
||||
setattr(
|
||||
router,
|
||||
registry_name,
|
||||
{"smart-route": [TaggedPreRoutingStrategy(tags=(), strategy=cls._RewriteStrategy())]},
|
||||
)
|
||||
return router
|
||||
|
||||
@staticmethod
|
||||
def _messages() -> list[dict[str, str]]:
|
||||
return [{"role": "user", "content": "What is the capital of France?"}]
|
||||
|
||||
@pytest.mark.parametrize("registry_name", REGISTRY_NAMES)
|
||||
@pytest.mark.asyncio
|
||||
async def test_alias_dispatches_to_the_strategy_registered_under_the_target(self, registry_name):
|
||||
from litellm.constants import AUTO_ROUTED_REQUEST_METADATA_KEY
|
||||
|
||||
router = self._router(registry_name)
|
||||
request_kwargs = {"metadata": {}}
|
||||
|
||||
response = await router.async_pre_routing_hook(
|
||||
model="smart-alias", request_kwargs=request_kwargs, messages=self._messages()
|
||||
)
|
||||
|
||||
assert response is not None
|
||||
assert response.model == "gemini-flash"
|
||||
assert request_kwargs["metadata"][AUTO_ROUTED_REQUEST_METADATA_KEY] is True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_alias_call_still_forwards_the_marker_own_params_to_the_routed_tier(self):
|
||||
router = self._router("auto_routers")
|
||||
request_kwargs = {"metadata": {}}
|
||||
|
||||
await router.async_pre_routing_hook(
|
||||
model="smart-alias", request_kwargs=request_kwargs, messages=self._messages()
|
||||
)
|
||||
|
||||
assert request_kwargs["timeout"] == self.MARKER_TIMEOUT
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_alias_deployment_selection_lands_on_the_tier_never_the_marker(self):
|
||||
router = self._router("auto_routers")
|
||||
|
||||
deployment = await router.async_get_available_deployment(
|
||||
model="smart-alias", request_kwargs={"metadata": {}}, messages=self._messages()
|
||||
)
|
||||
|
||||
assert deployment["litellm_params"]["model"] == "gemini/gemini-3.6-flash"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_alias_call_completes_and_still_bills_the_name_the_caller_sent(self):
|
||||
router = self._router("auto_routers")
|
||||
metadata: dict = {}
|
||||
|
||||
response = await router.acompletion(
|
||||
model="smart-alias", messages=self._messages(), metadata=metadata
|
||||
)
|
||||
|
||||
assert response.choices[0].message.content == "routed by the tier"
|
||||
assert metadata["model_group"] == "smart-alias"
|
||||
assert metadata["model_group_alias"] == "smart-alias"
|
||||
|
||||
def test_a_group_of_only_markers_is_not_a_callable_model(self):
|
||||
router = self._router(None)
|
||||
|
||||
with pytest.raises(litellm.BadRequestError, match="strategy router marker"):
|
||||
router.get_available_deployment(
|
||||
model="smart-route", messages=self._messages(), request_kwargs={"metadata": {}}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("local_model_cost_map")
|
||||
class TestAzureBaseModelFallbackLogging:
|
||||
"""When an azure deployment has no base_model but its model name is a known
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue