diff --git a/litellm/router.py b/litellm/router.py index 1f114f99118..c8441a6c93b 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -96,6 +96,7 @@ from litellm.router_utils.add_retry_fallback_headers import ( response_in_flight_token_count, ) from litellm.router_utils.auto_router_model_naming import ( + AUTO_ROUTER_MODEL_PREFIX, classify_strategy_router_model, ) from litellm.router_utils.batch_utils import ( @@ -318,6 +319,8 @@ def model_info_is_active_for_environment(model_info: Mapping[str, object] | None _PreRoutingStrategyT = TypeVar("_PreRoutingStrategyT") +_ALIAS_PARAMS_NEVER_FORWARDED: Final = frozenset({"model", "api_base", "api_key", "api_version"}) + def _stream_chunks_have_generated_content(chunks: Sequence[ModelResponseStream]) -> bool: for chunk in chunks: @@ -11360,7 +11363,9 @@ class Router: that share a `model_name` by matching the request's tags against each registered strategy's tags before falling back to the first registered. Returns the tagged registry entry so the caller can tell whether the - request's tags were what selected it. + request's tags were what selected it, and can locate the marker + deployment the strategy was registered from via its (model_name, tags) + pair. """ candidates: Final[list[TaggedPreRoutingStrategy[PreRoutingStrategy]]] = [ *self.auto_routers.get(model, []), @@ -11445,25 +11450,47 @@ class Router: ) # `model` (the alias, e.g. "smart-router") is never the deployment actually - # called - apply the alias's own litellm_params (besides `model` itself, - # which is just the alias marker) to the request, since the tier/route - # deployment the hook selected won't have them. Router-only fields - # (tpm, rpm, weight, complexity_router_config, ...) are excluded from the - # actual outbound LLM call downstream by litellm.types.utils.all_litellm_params, + # called - apply the router marker's own litellm_params to the request, + # since the tier/route deployment the hook selected won't have them. The + # marker entry is looked up by its `auto_router/` model prefix and the + # selected strategy's tags, never by list position: plain deployments may + # share the alias `model_name` and must not leak their params (`api_base`, + # `api_key`, ...) onto the routed call. Router-only fields (tpm, rpm, + # weight, complexity_router_config, ...) are excluded from the actual + # outbound LLM call downstream by litellm.types.utils.all_litellm_params, # not here. Custom pricing fields ARE call params, so they must be # excluded here: they price the alias, not the deployment the hook # selected, and forwarding them re-registers the routed deployment at # the alias's price (an explicit 0 makes every alias request bill $0). if pre_routing_hook_response is not None: - alias_index: Final = self.model_name_to_deployment_indices.get(model, []) - if alias_index: - alias_litellm_params: Final = self.model_list[alias_index[0]].get("litellm_params", {}) - for key, value in alias_litellm_params.items(): - if key != "model" and key not in CustomPricingLiteLLMParams.model_fields and value is not None: - request_kwargs.setdefault(key, value) + for key, value in self._forwardable_alias_marker_params(model=model, strategy_tags=selected_strategy.tags): + request_kwargs.setdefault(key, value) return pre_routing_hook_response + def _forwardable_alias_marker_params( + self, model: str, strategy_tags: tuple[str, ...] + ) -> tuple[tuple[str, object], ...]: + marker_params: Final = tuple( + litellm_params + for idx in self.model_name_to_deployment_indices.get(model, ()) + if isinstance(litellm_params := self.model_list[idx].get("litellm_params", {}), dict) + and str(litellm_params.get("model", "")).startswith(AUTO_ROUTER_MODEL_PREFIX) + ) + tag_matched: Final = tuple( + params for params in marker_params if tuple(params.get("tags") or ()) == strategy_tags + ) + selected: Final = tag_matched[0] if tag_matched else (marker_params[0] if marker_params else None) + if selected is None: + return () + return tuple( + (key, value) + for key, value in selected.items() + if key not in _ALIAS_PARAMS_NEVER_FORWARDED + and key not in CustomPricingLiteLLMParams.model_fields + and value is not None + ) + def _consumed_request_tags_stamp( self, selected_strategy: "TaggedPreRoutingStrategy[PreRoutingStrategy]", diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 7fc78aee619..ef0e5ac2229 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -2079,14 +2079,16 @@ class TestRouterPreRoutingAliasOverrides: assert field not in request_kwargs @pytest.mark.asyncio - async def test_alias_overrides_exclude_only_model(self): - """`model` (the alias marker, e.g. auto_router/complexity_router) is - excluded since it's never a real provider model. Router-only fields - like complexity_router_config DO flow through into request_kwargs at - this layer - they're filtered from the actual outbound LLM call - downstream by litellm.types.utils.all_litellm_params instead, not by - the router's pre-routing hook. See test_router_init_only_params_are_ - never_sent_to_a_provider for the guard on that downstream filter.""" + async def test_alias_overrides_exclude_only_marker_and_connection_params(self): + """`model` (the alias marker, e.g. auto_router/complexity_router) and + provider-connection params (api_base/api_key/api_version) are excluded + since they never describe the tier deployment actually called. + Router-only fields like complexity_router_config DO flow through into + request_kwargs at this layer - they're filtered from the actual + outbound LLM call downstream by litellm.types.utils.all_litellm_params + instead, not by the router's pre-routing hook. See + test_router_init_only_params_are_never_sent_to_a_provider for the + guard on that downstream filter.""" router = self._make_router() request_kwargs: Dict = {} @@ -2106,9 +2108,10 @@ class TestRouterPreRoutingAliasOverrides: assert request_kwargs["complexity_router_default_model"] == "gpt-4o" def test_router_init_only_params_are_never_sent_to_a_provider(self): - """The router's pre-routing hook only excludes `model` (see - test_alias_overrides_exclude_only_model above) - every other alias - litellm_param, including router-init-only fields like + """The router's pre-routing hook only excludes `model` and + provider-connection params (see test_alias_overrides_exclude_only_ + marker_and_connection_params above) - every other alias litellm_param, + including router-init-only fields like complexity_router_config, flows into request_kwargs unfiltered. That's only safe because litellm.completion()/acompletion() itself strips anything listed in all_litellm_params before building the provider @@ -2201,6 +2204,154 @@ class TestRouterPreRoutingAliasOverrides: assert request_kwargs["drop_params"] is True +class TestRouterPreRoutingSharedAliasName: + """ + Regression tests for https://github.com/BerriAI/litellm/issues/36619. + + A plain deployment and an `auto_router/` marker can share a `model_name`. + The alias-param forwarding after a pre-routing rewrite must read the + marker entry, never whichever same-name entry happens to sit first in + `model_list` - otherwise the plain entry's api_base/api_key get grafted + onto the routed tier's call (a Gemini path under api.openai.com, 404). + """ + + @staticmethod + def _plain_entry() -> dict: + return { + "model_name": "gpt4o", + "litellm_params": { + "model": "openai/gpt-4o", + "api_key": "sk-plain-entry", + "api_base": "https://plain-entry.example/v1", + }, + } + + @staticmethod + def _marker_entry() -> dict: + return { + "model_name": "gpt4o", + "litellm_params": { + "model": "auto_router/complexity_router", + "drop_params": True, + "complexity_router_config": {"tiers": {"SIMPLE": "gemini-flash", "MEDIUM": "gemini-flash"}}, + "complexity_router_default_model": "gemini-flash", + }, + } + + @staticmethod + def _tier_entry() -> dict: + return { + "model_name": "gemini-flash", + "litellm_params": {"model": "gemini/gemini-3.6-flash", "api_key": "sk-tier"}, + } + + @pytest.mark.asyncio + @pytest.mark.parametrize("plain_entry_first", [True, False], ids=["plain_entry_first", "marker_entry_first"]) + async def test_marker_params_forwarded_regardless_of_model_list_order(self, plain_entry_first): + """In either config order the routed call gets the marker's own params + (drop_params) and never the plain sibling's api_base/api_key.""" + shared_name_entries = ( + [self._plain_entry(), self._marker_entry()] + if plain_entry_first + else [self._marker_entry(), self._plain_entry()] + ) + router = Router(model_list=[*shared_name_entries, self._tier_entry()]) + request_kwargs: Dict = {} + + result = await router.async_pre_routing_hook( + model="gpt4o", + request_kwargs=request_kwargs, + messages=[{"role": "user", "content": "What is the capital of France?"}], + ) + + assert result is not None + assert result.model == "gemini-flash" + assert "api_base" not in request_kwargs + assert "api_key" not in request_kwargs + assert request_kwargs["drop_params"] is True + + @pytest.mark.asyncio + async def test_connection_params_on_the_marker_itself_are_not_forwarded(self): + """Even when the marker entry carries api_base/api_key/api_version, + they describe no real deployment and must not reach the routed call, + while the marker's other params still do.""" + marker_with_connection_params = { + "model_name": "smart", + "litellm_params": { + **self._marker_entry()["litellm_params"], + "api_key": "sk-marker", + "api_base": "https://marker.example/v1", + "api_version": "2024-01-01", + }, + } + router = Router(model_list=[marker_with_connection_params, self._tier_entry()]) + request_kwargs: Dict = {} + + result = await router.async_pre_routing_hook( + model="smart", + request_kwargs=request_kwargs, + messages=[{"role": "user", "content": "hi"}], + ) + + assert result is not None + assert "api_base" not in request_kwargs + assert "api_key" not in request_kwargs + assert "api_version" not in request_kwargs + assert request_kwargs["drop_params"] is True + + @pytest.mark.asyncio + async def test_tag_scoped_markers_forward_the_selected_markers_params(self): + """With two tag-scoped markers under one name, the forwarded params + come from the marker whose tags matched the request, not from the + first marker in the list.""" + + def tagged_marker(routed_model: str, tags: list, drop_params: bool | None) -> dict: + return { + "model_name": "smart", + "litellm_params": { + "model": "auto_router/complexity_router", + "complexity_router_default_model": routed_model, + "complexity_router_config": {"tiers": {"SIMPLE": [routed_model], "MEDIUM": [routed_model]}}, + "tags": tags, + **({"drop_params": drop_params} if drop_params is not None else {}), + }, + } + + router = Router( + model_list=[ + tagged_marker("gpt-cn", ["cn"], None), + tagged_marker("gpt-us", ["us"], True), + ] + ) + + us_kwargs: Dict = {"metadata": {"tags": ["us"]}} + us_result = await router.async_pre_routing_hook( + model="smart", + request_kwargs=us_kwargs, + messages=[{"role": "user", "content": "hi"}], + ) + assert us_result is not None and us_result.model == "gpt-us" + assert us_kwargs["drop_params"] is True + + cn_kwargs: Dict = {"metadata": {"tags": ["cn"]}} + cn_result = await router.async_pre_routing_hook( + model="smart", + request_kwargs=cn_kwargs, + messages=[{"role": "user", "content": "hi"}], + ) + assert cn_result is not None and cn_result.model == "gpt-cn" + assert "drop_params" not in cn_kwargs + + def test_forwardable_alias_marker_params_reads_the_marker_entry_only(self): + router = Router(model_list=[self._plain_entry(), self._marker_entry(), self._tier_entry()]) + + forwarded = dict(router._forwardable_alias_marker_params(model="gpt4o", strategy_tags=())) + + assert forwarded["drop_params"] is True + assert "api_key" not in forwarded and "api_base" not in forwarded + assert router._forwardable_alias_marker_params(model="gemini-flash", strategy_tags=()) == () + + class TestAdaptiveSoftFloors: def test_adaptive_defaults_use_cost_weighted_cold_policy(self): config = ComplexityRouterConfig(