From 2604f888c484ff3e1d7bb1f2ad4ff9c7282e15f7 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 20 Aug 2026 03:03:10 -0700 Subject: [PATCH] fix(router): let the routed deployment's own litellm_params beat forwarded auto_router marker params An `auto_router/` marker entry's litellm_params (for example `aws_region_name: eu-west-3`) were forwarded onto every routed call with setdefault and then won the `{**litellm_params, **kwargs}` merge against the selected tier's own values, so a Bedrock tier pinned to us-east-1 was called in eu-west-3 and failed with 400. The hook now records which keys it actually forwarded on the request's metadata bucket, and `_update_kwargs_with_deployment` drops every forwarded key the selected deployment defines itself, so marker params only fill gaps a tier leaves open. Request-supplied values still win over both. The stamp is stripped from logged metadata like its siblings. Fixes #37613 --- litellm/constants.py | 1 + litellm/proxy/common_utils/callback_utils.py | 2 + litellm/proxy/litellm_pre_call_utils.py | 2 + litellm/router.py | 51 ++++++++++- .../router_strategy/test_complexity_router.py | 88 +++++++++++++++++++ 5 files changed, 141 insertions(+), 3 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index facfc6f7c19..ea1c3f9ecee 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1329,6 +1329,7 @@ OLD_LITELLM_METADATA_FIELD: Final = "metadata" RETURN_RAW_MODEL_NAME_METADATA_KEY: Final = "_complexity_router_return_raw_model_name" SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY: Final = "_session_deployment_affinity_ttl" CONSUMED_REQUEST_TAGS_METADATA_KEY: Final = "_consumed_request_tags" +ALIAS_MARKER_FORWARDED_PARAMS_METADATA_KEY: Final = "_alias_marker_forwarded_params" INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin" LITELLM_TRUNCATED_PAYLOAD_FIELD: Final = "litellm_truncated" LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE: Final = ( diff --git a/litellm/proxy/common_utils/callback_utils.py b/litellm/proxy/common_utils/callback_utils.py index 9379a8577a3..b6a0683c0ff 100644 --- a/litellm/proxy/common_utils/callback_utils.py +++ b/litellm/proxy/common_utils/callback_utils.py @@ -10,6 +10,7 @@ import litellm from litellm import get_secret from litellm._logging import verbose_proxy_logger from litellm.constants import ( + ALIAS_MARKER_FORWARDED_PARAMS_METADATA_KEY, CONSUMED_REQUEST_TAGS_METADATA_KEY, PRE_CALL_EXECUTED_GUARDRAILS_KEY, SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, @@ -507,6 +508,7 @@ LITELLM_PROXY_INTERNAL_METADATA_KEYS: Final = frozenset( PRE_CALL_EXECUTED_GUARDRAILS_KEY, SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, CONSUMED_REQUEST_TAGS_METADATA_KEY, + ALIAS_MARKER_FORWARDED_PARAMS_METADATA_KEY, "disable_global_guardrails", "disable_global_guardrail", "opted_out_global_guardrails", diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 2ec5c34958c..590852d5e8d 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -16,6 +16,7 @@ import litellm from litellm._logging import verbose_logger, verbose_proxy_logger from litellm._service_logger import ServiceLogging from litellm.constants import ( + ALIAS_MARKER_FORWARDED_PARAMS_METADATA_KEY, CONSUMED_REQUEST_TAGS_METADATA_KEY, INTERNAL_CALL_ORIGIN_METADATA_KEY, LITELLM_PROXY_MASTER_KEY_ALIAS, @@ -267,6 +268,7 @@ _UNTRUSTED_METADATA_CONTROL_FIELDS: Final = ( "routing_decision", SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, CONSUMED_REQUEST_TAGS_METADATA_KEY, + ALIAS_MARKER_FORWARDED_PARAMS_METADATA_KEY, INTERNAL_CALL_ORIGIN_METADATA_KEY, "standard_logging_object", "proxy_server_request", diff --git a/litellm/router.py b/litellm/router.py index 26158ae0a56..f8fc5bede8e 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -44,6 +44,7 @@ from litellm.caching.caching import ( RedisClusterCache, ) from litellm.constants import ( + ALIAS_MARKER_FORWARDED_PARAMS_METADATA_KEY, CONSUMED_REQUEST_TAGS_METADATA_KEY, DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS, DEFAULT_HEALTH_CHECK_INTERVAL, @@ -3239,6 +3240,8 @@ class Router: - Adds default litellm params to kwargs, if set. - Merges tools from deployment with request (proxy-configured tools + request tools). """ + for key in self._forwarded_alias_marker_keys_the_deployment_sets(deployment=deployment, kwargs=kwargs): + kwargs.pop(key, None) self._merge_tools_from_deployment(deployment=deployment, kwargs=kwargs) model_info = deployment.get("model_info", {}).copy() @@ -11564,6 +11567,11 @@ class Router: self._stamp_or_clear_metadata_key( request_kwargs=request_kwargs, key=CONSUMED_REQUEST_TAGS_METADATA_KEY, value=None ) + self._stamp_or_clear_metadata_key( + request_kwargs=request_kwargs, + key=ALIAS_MARKER_FORWARDED_PARAMS_METADATA_KEY, + value=self._still_forwarded_alias_marker_keys(request_kwargs) or None, + ) return None pre_routing_hook_response: Final = await selected_strategy.strategy.async_pre_routing_hook( @@ -11605,9 +11613,22 @@ class Router: # excluded here: they price the alias, not the deployment the hook # selected, and forwarding them re-registers the routed deployment at # the alias's price (an explicit 0 makes every alias request bill $0). - if pre_routing_hook_response is not None: - for key, value in self._forwardable_alias_marker_params(model=model, strategy_tags=selected_strategy.tags): - request_kwargs.setdefault(key, value) + # Forwarded params only fill gaps: the keys inserted here are stamped on the + # request so `_update_kwargs_with_deployment` can drop any the selected + # deployment sets itself (its own `aws_region_name` beats the marker's). + marker_params: Final = ( + self._forwardable_alias_marker_params(model=model, strategy_tags=selected_strategy.tags) + if pre_routing_hook_response is not None + else () + ) + still_forwarded: Final = self._still_forwarded_alias_marker_keys(request_kwargs) + newly_forwarded: Final = tuple((key, value) for key, value in marker_params if key not in request_kwargs) + request_kwargs.update(newly_forwarded) + self._stamp_or_clear_metadata_key( + request_kwargs=request_kwargs, + key=ALIAS_MARKER_FORWARDED_PARAMS_METADATA_KEY, + value=(*still_forwarded, *(key for key, _ in newly_forwarded)) or None, + ) return pre_routing_hook_response @@ -11634,6 +11655,30 @@ class Router: and value is not None ) + @staticmethod + def _still_forwarded_alias_marker_keys(request_kwargs: Mapping[str, object]) -> tuple[str, ...]: + stamps: Final = tuple( + stamp + for bucket in (request_kwargs.get("litellm_metadata"), request_kwargs.get("metadata")) + if isinstance(bucket, Mapping) + for stamp in (bucket.get(ALIAS_MARKER_FORWARDED_PARAMS_METADATA_KEY),) + if isinstance(stamp, (list, tuple)) + ) + return tuple(key for stamp in stamps for key in stamp if isinstance(key, str) and key in request_kwargs) + + @staticmethod + def _forwarded_alias_marker_keys_the_deployment_sets( + deployment: Mapping[str, object], kwargs: Mapping[str, object] + ) -> tuple[str, ...]: + deployment_litellm_params: Final = deployment.get("litellm_params") + if not isinstance(deployment_litellm_params, Mapping): + return () + return tuple( + key + for key in Router._still_forwarded_alias_marker_keys(kwargs) + if deployment_litellm_params.get(key) is not None + ) + def _consumed_request_tags_stamp( self, selected_strategy: "TaggedPreRoutingStrategy[PreRoutingStrategy]", diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 006fb79b08f..ac3b1089e3f 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -2509,6 +2509,94 @@ class TestRouterPreRoutingSharedAliasName: assert "api_key" not in forwarded and "api_base" not in forwarded assert router._forwardable_alias_marker_params(model="gemini-flash", strategy_tags=()) == () + @staticmethod + def _region_marker_entry() -> dict: + return { + "model_name": "smart-router", + "litellm_params": { + "model": "auto_router/complexity_router", + "aws_region_name": "eu-west-3", + "drop_params": True, + "complexity_router_config": {"tiers": {"SIMPLE": "bedrock-tier", "MEDIUM": "bedrock-tier"}}, + "complexity_router_default_model": "bedrock-tier", + }, + } + + @staticmethod + def _bedrock_tier_entry(model_name: str = "bedrock-tier", aws_region_name: str | None = None) -> dict: + return { + "model_name": model_name, + "litellm_params": { + "model": "bedrock/us.anthropic.claude-sonnet-5", + **({"aws_region_name": aws_region_name} if aws_region_name else {}), + }, + } + + @staticmethod + async def _routed_call_kwargs(router: Router, **request_params) -> dict: + mock_acompletion = AsyncMock(return_value=litellm.ModelResponse(choices=[{"message": {"content": "hi"}}])) + with patch.object(litellm, "acompletion", mock_acompletion): + await router.acompletion( + model="smart-router", messages=[{"role": "user", "content": "hi"}], **request_params + ) + return mock_acompletion.call_args.kwargs + + @pytest.mark.asyncio + async def test_tier_deployments_own_params_beat_the_markers_forwarded_params(self): + """A marker-level `aws_region_name` only fills the gap for tiers that set none: + a tier pinned to its own region must be called there, not in the marker's.""" + router = Router(model_list=[self._region_marker_entry(), self._bedrock_tier_entry(aws_region_name="us-east-1")]) + + sent = await self._routed_call_kwargs(router) + + assert sent["model"] == "bedrock/us.anthropic.claude-sonnet-5" + assert sent["aws_region_name"] == "us-east-1" + assert sent["drop_params"] is True + + @pytest.mark.asyncio + async def test_marker_params_still_fill_the_gaps_a_tier_leaves_open(self): + router = Router(model_list=[self._region_marker_entry(), self._bedrock_tier_entry()]) + + sent = await self._routed_call_kwargs(router) + + assert sent["aws_region_name"] == "eu-west-3" + assert sent["drop_params"] is True + + @pytest.mark.asyncio + async def test_request_supplied_param_beats_both_the_marker_and_the_tier(self): + router = Router(model_list=[self._region_marker_entry(), self._bedrock_tier_entry(aws_region_name="us-east-1")]) + + sent = await self._routed_call_kwargs(router, aws_region_name="ap-south-1") + + assert sent["aws_region_name"] == "ap-south-1" + + @pytest.mark.asyncio + async def test_forwarded_params_keep_yielding_to_the_deployment_on_later_routing_passes(self): + """A retry or fallback re-enters the hook with the first pass's forwarded params + already in the kwargs; they must still yield to the next deployment's own values.""" + router = Router( + model_list=[ + self._region_marker_entry(), + self._bedrock_tier_entry(), + self._bedrock_tier_entry(model_name="pinned-tier", aws_region_name="us-east-1"), + ] + ) + request_kwargs: Dict = {} + messages = [{"role": "user", "content": "hi"}] + + await router.async_pre_routing_hook(model="smart-router", request_kwargs=request_kwargs, messages=messages) + router._update_kwargs_with_deployment( + deployment=router.get_model_list(model_name="bedrock-tier")[0], kwargs=request_kwargs + ) + assert request_kwargs["aws_region_name"] == "eu-west-3" + + await router.async_pre_routing_hook(model="smart-router", request_kwargs=request_kwargs, messages=messages) + router._update_kwargs_with_deployment( + deployment=router.get_model_list(model_name="pinned-tier")[0], kwargs=request_kwargs + ) + assert "aws_region_name" not in request_kwargs + assert request_kwargs["drop_params"] is True + class TestAdaptiveSoftFloors: def test_adaptive_defaults_use_cost_weighted_cold_policy(self):