scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-08-28 05:25:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 36

Merge pull request #36626 from BerriAI/litellm_fix_autorouter_alias_forwarding

Browse source 
fix(router): forward auto-router alias params from the marker entry, not the first same-name deployment
This commit is contained in:
Mateo Wang 2026-08-12 16:55:12 -07:00 • committed by GitHub
commit 3b5317c40d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 201 additions and 23 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

51
litellm/router.py
View file

@ -96,6 +96,7 @@ from litellm.router_utils.add_retry_fallback_headers import (
response_in_flight_token_count,
)
from litellm.router_utils.auto_router_model_naming import (
AUTO_ROUTER_MODEL_PREFIX,
classify_strategy_router_model,
)
from litellm.router_utils.batch_utils import (
@ -318,6 +319,8 @@ def model_info_is_active_for_environment(model_info: Mapping[str, object] | None
_PreRoutingStrategyT = TypeVar("_PreRoutingStrategyT")
_ALIAS_PARAMS_NEVER_FORWARDED: Final = frozenset({"model", "api_base", "api_key", "api_version"})
def _stream_chunks_have_generated_content(chunks: Sequence[ModelResponseStream]) -> bool:
for chunk in chunks:
@ -11360,7 +11363,9 @@ class Router:
that share a `model_name` by matching the request's tags against each
registered strategy's tags before falling back to the first registered.
Returns the tagged registry entry so the caller can tell whether the
request's tags were what selected it.
request's tags were what selected it, and can locate the marker
deployment the strategy was registered from via its (model_name, tags)
pair.
"""
candidates: Final[list[TaggedPreRoutingStrategy[PreRoutingStrategy]]] = [
*self.auto_routers.get(model, []),
@ -11445,25 +11450,47 @@ class Router:
)
# `model` (the alias, e.g. "smart-router") is never the deployment actually
# called - apply the alias's own litellm_params (besides `model` itself,
# which is just the alias marker) to the request, since the tier/route
# deployment the hook selected won't have them. Router-only fields
# (tpm, rpm, weight, complexity_router_config, ...) are excluded from the
# actual outbound LLM call downstream by litellm.types.utils.all_litellm_params,
# called - apply the router marker's own litellm_params to the request,
# since the tier/route deployment the hook selected won't have them. The
# marker entry is looked up by its `auto_router/` model prefix and the
# selected strategy's tags, never by list position: plain deployments may
# share the alias `model_name` and must not leak their params (`api_base`,
# `api_key`, ...) onto the routed call. Router-only fields (tpm, rpm,
# weight, complexity_router_config, ...) are excluded from the actual
# outbound LLM call downstream by litellm.types.utils.all_litellm_params,
# not here. Custom pricing fields ARE call params, so they must be
# excluded here: they price the alias, not the deployment the hook
# selected, and forwarding them re-registers the routed deployment at
# the alias's price (an explicit 0 makes every alias request bill $0).
if pre_routing_hook_response is not None:
alias_index: Final = self.model_name_to_deployment_indices.get(model, [])
if alias_index:
alias_litellm_params: Final = self.model_list[alias_index[0]].get("litellm_params", {})
for key, value in alias_litellm_params.items():
if key != "model" and key not in CustomPricingLiteLLMParams.model_fields and value is not None:
request_kwargs.setdefault(key, value)
for key, value in self._forwardable_alias_marker_params(model=model, strategy_tags=selected_strategy.tags):
request_kwargs.setdefault(key, value)
return pre_routing_hook_response
def _forwardable_alias_marker_params(
self, model: str, strategy_tags: tuple[str, ...]
) -> tuple[tuple[str, object], ...]:
marker_params: Final = tuple(
litellm_params
for idx in self.model_name_to_deployment_indices.get(model, ())
if isinstance(litellm_params := self.model_list[idx].get("litellm_params", {}), dict)
and str(litellm_params.get("model", "")).startswith(AUTO_ROUTER_MODEL_PREFIX)
)
tag_matched: Final = tuple(
params for params in marker_params if tuple(params.get("tags") or ()) == strategy_tags
)
selected: Final = tag_matched[0] if tag_matched else (marker_params[0] if marker_params else None)
if selected is None:
return ()
return tuple(
(key, value)
for key, value in selected.items()
if key not in _ALIAS_PARAMS_NEVER_FORWARDED
and key not in CustomPricingLiteLLMParams.model_fields
and value is not None
)
def _consumed_request_tags_stamp(
self,
selected_strategy: "TaggedPreRoutingStrategy[PreRoutingStrategy]",

173
tests/test_litellm/router_strategy/test_complexity_router.py
View file

@ -2079,14 +2079,16 @@ class TestRouterPreRoutingAliasOverrides:
assert field not in request_kwargs
@pytest.mark.asyncio
async def test_alias_overrides_exclude_only_model(self):
"""`model` (the alias marker, e.g. auto_router/complexity_router) is
excluded since it's never a real provider model. Router-only fields
like complexity_router_config DO flow through into request_kwargs at
this layer - they're filtered from the actual outbound LLM call
downstream by litellm.types.utils.all_litellm_params instead, not by
the router's pre-routing hook. See test_router_init_only_params_are_
never_sent_to_a_provider for the guard on that downstream filter."""
async def test_alias_overrides_exclude_only_marker_and_connection_params(self):
"""`model` (the alias marker, e.g. auto_router/complexity_router) and
provider-connection params (api_base/api_key/api_version) are excluded
since they never describe the tier deployment actually called.
Router-only fields like complexity_router_config DO flow through into
request_kwargs at this layer - they're filtered from the actual
outbound LLM call downstream by litellm.types.utils.all_litellm_params
instead, not by the router's pre-routing hook. See
test_router_init_only_params_are_never_sent_to_a_provider for the
guard on that downstream filter."""
router = self._make_router()
request_kwargs: Dict = {}
@ -2106,9 +2108,10 @@ class TestRouterPreRoutingAliasOverrides:
assert request_kwargs["complexity_router_default_model"] == "gpt-4o"
def test_router_init_only_params_are_never_sent_to_a_provider(self):
"""The router's pre-routing hook only excludes `model` (see
test_alias_overrides_exclude_only_model above) - every other alias
litellm_param, including router-init-only fields like
"""The router's pre-routing hook only excludes `model` and
provider-connection params (see test_alias_overrides_exclude_only_
marker_and_connection_params above) - every other alias litellm_param,
including router-init-only fields like
complexity_router_config, flows into request_kwargs unfiltered. That's
only safe because litellm.completion()/acompletion() itself strips
anything listed in all_litellm_params before building the provider
@ -2201,6 +2204,154 @@ class TestRouterPreRoutingAliasOverrides:
assert request_kwargs["drop_params"] is True
class TestRouterPreRoutingSharedAliasName:
"""
Regression tests for https://github.com/BerriAI/litellm/issues/36619.
A plain deployment and an `auto_router/` marker can share a `model_name`.
The alias-param forwarding after a pre-routing rewrite must read the
marker entry, never whichever same-name entry happens to sit first in
`model_list` - otherwise the plain entry's api_base/api_key get grafted
onto the routed tier's call (a Gemini path under api.openai.com, 404).
"""
@staticmethod
def _plain_entry() -> dict:
return {
"model_name": "gpt4o",
"litellm_params": {
"model": "openai/gpt-4o",
"api_key": "sk-plain-entry",
"api_base": "https://plain-entry.example/v1",
},
}
@staticmethod
def _marker_entry() -> dict:
return {
"model_name": "gpt4o",
"litellm_params": {
"model": "auto_router/complexity_router",
"drop_params": True,
"complexity_router_config": {"tiers": {"SIMPLE": "gemini-flash", "MEDIUM": "gemini-flash"}},
"complexity_router_default_model": "gemini-flash",
},
}
@staticmethod
def _tier_entry() -> dict:
return {
"model_name": "gemini-flash",
"litellm_params": {"model": "gemini/gemini-3.6-flash", "api_key": "sk-tier"},
}
@pytest.mark.asyncio
@pytest.mark.parametrize("plain_entry_first", [True, False], ids=["plain_entry_first", "marker_entry_first"])
async def test_marker_params_forwarded_regardless_of_model_list_order(self, plain_entry_first):
"""In either config order the routed call gets the marker's own params
(drop_params) and never the plain sibling's api_base/api_key."""
shared_name_entries = (
[self._plain_entry(), self._marker_entry()]
if plain_entry_first
else [self._marker_entry(), self._plain_entry()]
)
router = Router(model_list=[*shared_name_entries, self._tier_entry()])
request_kwargs: Dict = {}
result = await router.async_pre_routing_hook(
model="gpt4o",
request_kwargs=request_kwargs,
messages=[{"role": "user", "content": "What is the capital of France?"}],
)
assert result is not None
assert result.model == "gemini-flash"
assert "api_base" not in request_kwargs
assert "api_key" not in request_kwargs
assert request_kwargs["drop_params"] is True
@pytest.mark.asyncio
async def test_connection_params_on_the_marker_itself_are_not_forwarded(self):
"""Even when the marker entry carries api_base/api_key/api_version,
they describe no real deployment and must not reach the routed call,
while the marker's other params still do."""
marker_with_connection_params = {
"model_name": "smart",
"litellm_params": {
**self._marker_entry()["litellm_params"],
"api_key": "sk-marker",
"api_base": "https://marker.example/v1",
"api_version": "2024-01-01",
},
}
router = Router(model_list=[marker_with_connection_params, self._tier_entry()])
request_kwargs: Dict = {}
result = await router.async_pre_routing_hook(
model="smart",
request_kwargs=request_kwargs,
messages=[{"role": "user", "content": "hi"}],
)
assert result is not None
assert "api_base" not in request_kwargs
assert "api_key" not in request_kwargs
assert "api_version" not in request_kwargs
assert request_kwargs["drop_params"] is True
@pytest.mark.asyncio
async def test_tag_scoped_markers_forward_the_selected_markers_params(self):
"""With two tag-scoped markers under one name, the forwarded params
come from the marker whose tags matched the request, not from the
first marker in the list."""
def tagged_marker(routed_model: str, tags: list, drop_params: bool | None) -> dict:
return {
"model_name": "smart",
"litellm_params": {
"model": "auto_router/complexity_router",
"complexity_router_default_model": routed_model,
"complexity_router_config": {"tiers": {"SIMPLE": [routed_model], "MEDIUM": [routed_model]}},
"tags": tags,
**({"drop_params": drop_params} if drop_params is not None else {}),
},
}
router = Router(
model_list=[
tagged_marker("gpt-cn", ["cn"], None),
tagged_marker("gpt-us", ["us"], True),
]
)
us_kwargs: Dict = {"metadata": {"tags": ["us"]}}
us_result = await router.async_pre_routing_hook(
model="smart",
request_kwargs=us_kwargs,
messages=[{"role": "user", "content": "hi"}],
)
assert us_result is not None and us_result.model == "gpt-us"
assert us_kwargs["drop_params"] is True
cn_kwargs: Dict = {"metadata": {"tags": ["cn"]}}
cn_result = await router.async_pre_routing_hook(
model="smart",
request_kwargs=cn_kwargs,
messages=[{"role": "user", "content": "hi"}],
)
assert cn_result is not None and cn_result.model == "gpt-cn"
assert "drop_params" not in cn_kwargs
def test_forwardable_alias_marker_params_reads_the_marker_entry_only(self):
router = Router(model_list=[self._plain_entry(), self._marker_entry(), self._tier_entry()])
forwarded = dict(router._forwardable_alias_marker_params(model="gpt4o", strategy_tags=()))
assert forwarded["drop_params"] is True
assert "api_key" not in forwarded and "api_base" not in forwarded
assert router._forwardable_alias_marker_params(model="gemini-flash", strategy_tags=()) == ()
class TestAdaptiveSoftFloors:
def test_adaptive_defaults_use_cost_weighted_cold_policy(self):
config = ComplexityRouterConfig(