From 2604f888c484ff3e1d7bb1f2ad4ff9c7282e15f7 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 20 Aug 2026 03:03:10 -0700 Subject: [PATCH 1/2] fix(router): let the routed deployment's own litellm_params beat forwarded auto_router marker params An `auto_router/` marker entry's litellm_params (for example `aws_region_name: eu-west-3`) were forwarded onto every routed call with setdefault and then won the `{**litellm_params, **kwargs}` merge against the selected tier's own values, so a Bedrock tier pinned to us-east-1 was called in eu-west-3 and failed with 400. The hook now records which keys it actually forwarded on the request's metadata bucket, and `_update_kwargs_with_deployment` drops every forwarded key the selected deployment defines itself, so marker params only fill gaps a tier leaves open. Request-supplied values still win over both. The stamp is stripped from logged metadata like its siblings. Fixes #37613 --- litellm/constants.py | 1 + litellm/proxy/common_utils/callback_utils.py | 2 + litellm/proxy/litellm_pre_call_utils.py | 2 + litellm/router.py | 51 ++++++++++- .../router_strategy/test_complexity_router.py | 88 +++++++++++++++++++ 5 files changed, 141 insertions(+), 3 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index facfc6f7c19..ea1c3f9ecee 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1329,6 +1329,7 @@ OLD_LITELLM_METADATA_FIELD: Final = "metadata" RETURN_RAW_MODEL_NAME_METADATA_KEY: Final = "_complexity_router_return_raw_model_name" SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY: Final = "_session_deployment_affinity_ttl" CONSUMED_REQUEST_TAGS_METADATA_KEY: Final = "_consumed_request_tags" +ALIAS_MARKER_FORWARDED_PARAMS_METADATA_KEY: Final = "_alias_marker_forwarded_params" INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin" LITELLM_TRUNCATED_PAYLOAD_FIELD: Final = "litellm_truncated" LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE: Final = ( diff --git a/litellm/proxy/common_utils/callback_utils.py b/litellm/proxy/common_utils/callback_utils.py index 9379a8577a3..b6a0683c0ff 100644 --- a/litellm/proxy/common_utils/callback_utils.py +++ b/litellm/proxy/common_utils/callback_utils.py @@ -10,6 +10,7 @@ import litellm from litellm import get_secret from litellm._logging import verbose_proxy_logger from litellm.constants import ( + ALIAS_MARKER_FORWARDED_PARAMS_METADATA_KEY, CONSUMED_REQUEST_TAGS_METADATA_KEY, PRE_CALL_EXECUTED_GUARDRAILS_KEY, SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, @@ -507,6 +508,7 @@ LITELLM_PROXY_INTERNAL_METADATA_KEYS: Final = frozenset( PRE_CALL_EXECUTED_GUARDRAILS_KEY, SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, CONSUMED_REQUEST_TAGS_METADATA_KEY, + ALIAS_MARKER_FORWARDED_PARAMS_METADATA_KEY, "disable_global_guardrails", "disable_global_guardrail", "opted_out_global_guardrails", diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 2ec5c34958c..590852d5e8d 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -16,6 +16,7 @@ import litellm from litellm._logging import verbose_logger, verbose_proxy_logger from litellm._service_logger import ServiceLogging from litellm.constants import ( + ALIAS_MARKER_FORWARDED_PARAMS_METADATA_KEY, CONSUMED_REQUEST_TAGS_METADATA_KEY, INTERNAL_CALL_ORIGIN_METADATA_KEY, LITELLM_PROXY_MASTER_KEY_ALIAS, @@ -267,6 +268,7 @@ _UNTRUSTED_METADATA_CONTROL_FIELDS: Final = ( "routing_decision", SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, CONSUMED_REQUEST_TAGS_METADATA_KEY, + ALIAS_MARKER_FORWARDED_PARAMS_METADATA_KEY, INTERNAL_CALL_ORIGIN_METADATA_KEY, "standard_logging_object", "proxy_server_request", diff --git a/litellm/router.py b/litellm/router.py index 26158ae0a56..f8fc5bede8e 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -44,6 +44,7 @@ from litellm.caching.caching import ( RedisClusterCache, ) from litellm.constants import ( + ALIAS_MARKER_FORWARDED_PARAMS_METADATA_KEY, CONSUMED_REQUEST_TAGS_METADATA_KEY, DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS, DEFAULT_HEALTH_CHECK_INTERVAL, @@ -3239,6 +3240,8 @@ class Router: - Adds default litellm params to kwargs, if set. - Merges tools from deployment with request (proxy-configured tools + request tools). """ + for key in self._forwarded_alias_marker_keys_the_deployment_sets(deployment=deployment, kwargs=kwargs): + kwargs.pop(key, None) self._merge_tools_from_deployment(deployment=deployment, kwargs=kwargs) model_info = deployment.get("model_info", {}).copy() @@ -11564,6 +11567,11 @@ class Router: self._stamp_or_clear_metadata_key( request_kwargs=request_kwargs, key=CONSUMED_REQUEST_TAGS_METADATA_KEY, value=None ) + self._stamp_or_clear_metadata_key( + request_kwargs=request_kwargs, + key=ALIAS_MARKER_FORWARDED_PARAMS_METADATA_KEY, + value=self._still_forwarded_alias_marker_keys(request_kwargs) or None, + ) return None pre_routing_hook_response: Final = await selected_strategy.strategy.async_pre_routing_hook( @@ -11605,9 +11613,22 @@ class Router: # excluded here: they price the alias, not the deployment the hook # selected, and forwarding them re-registers the routed deployment at # the alias's price (an explicit 0 makes every alias request bill $0). - if pre_routing_hook_response is not None: - for key, value in self._forwardable_alias_marker_params(model=model, strategy_tags=selected_strategy.tags): - request_kwargs.setdefault(key, value) + # Forwarded params only fill gaps: the keys inserted here are stamped on the + # request so `_update_kwargs_with_deployment` can drop any the selected + # deployment sets itself (its own `aws_region_name` beats the marker's). + marker_params: Final = ( + self._forwardable_alias_marker_params(model=model, strategy_tags=selected_strategy.tags) + if pre_routing_hook_response is not None + else () + ) + still_forwarded: Final = self._still_forwarded_alias_marker_keys(request_kwargs) + newly_forwarded: Final = tuple((key, value) for key, value in marker_params if key not in request_kwargs) + request_kwargs.update(newly_forwarded) + self._stamp_or_clear_metadata_key( + request_kwargs=request_kwargs, + key=ALIAS_MARKER_FORWARDED_PARAMS_METADATA_KEY, + value=(*still_forwarded, *(key for key, _ in newly_forwarded)) or None, + ) return pre_routing_hook_response @@ -11634,6 +11655,30 @@ class Router: and value is not None ) + @staticmethod + def _still_forwarded_alias_marker_keys(request_kwargs: Mapping[str, object]) -> tuple[str, ...]: + stamps: Final = tuple( + stamp + for bucket in (request_kwargs.get("litellm_metadata"), request_kwargs.get("metadata")) + if isinstance(bucket, Mapping) + for stamp in (bucket.get(ALIAS_MARKER_FORWARDED_PARAMS_METADATA_KEY),) + if isinstance(stamp, (list, tuple)) + ) + return tuple(key for stamp in stamps for key in stamp if isinstance(key, str) and key in request_kwargs) + + @staticmethod + def _forwarded_alias_marker_keys_the_deployment_sets( + deployment: Mapping[str, object], kwargs: Mapping[str, object] + ) -> tuple[str, ...]: + deployment_litellm_params: Final = deployment.get("litellm_params") + if not isinstance(deployment_litellm_params, Mapping): + return () + return tuple( + key + for key in Router._still_forwarded_alias_marker_keys(kwargs) + if deployment_litellm_params.get(key) is not None + ) + def _consumed_request_tags_stamp( self, selected_strategy: "TaggedPreRoutingStrategy[PreRoutingStrategy]", diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 006fb79b08f..ac3b1089e3f 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -2509,6 +2509,94 @@ class TestRouterPreRoutingSharedAliasName: assert "api_key" not in forwarded and "api_base" not in forwarded assert router._forwardable_alias_marker_params(model="gemini-flash", strategy_tags=()) == () + @staticmethod + def _region_marker_entry() -> dict: + return { + "model_name": "smart-router", + "litellm_params": { + "model": "auto_router/complexity_router", + "aws_region_name": "eu-west-3", + "drop_params": True, + "complexity_router_config": {"tiers": {"SIMPLE": "bedrock-tier", "MEDIUM": "bedrock-tier"}}, + "complexity_router_default_model": "bedrock-tier", + }, + } + + @staticmethod + def _bedrock_tier_entry(model_name: str = "bedrock-tier", aws_region_name: str | None = None) -> dict: + return { + "model_name": model_name, + "litellm_params": { + "model": "bedrock/us.anthropic.claude-sonnet-5", + **({"aws_region_name": aws_region_name} if aws_region_name else {}), + }, + } + + @staticmethod + async def _routed_call_kwargs(router: Router, **request_params) -> dict: + mock_acompletion = AsyncMock(return_value=litellm.ModelResponse(choices=[{"message": {"content": "hi"}}])) + with patch.object(litellm, "acompletion", mock_acompletion): + await router.acompletion( + model="smart-router", messages=[{"role": "user", "content": "hi"}], **request_params + ) + return mock_acompletion.call_args.kwargs + + @pytest.mark.asyncio + async def test_tier_deployments_own_params_beat_the_markers_forwarded_params(self): + """A marker-level `aws_region_name` only fills the gap for tiers that set none: + a tier pinned to its own region must be called there, not in the marker's.""" + router = Router(model_list=[self._region_marker_entry(), self._bedrock_tier_entry(aws_region_name="us-east-1")]) + + sent = await self._routed_call_kwargs(router) + + assert sent["model"] == "bedrock/us.anthropic.claude-sonnet-5" + assert sent["aws_region_name"] == "us-east-1" + assert sent["drop_params"] is True + + @pytest.mark.asyncio + async def test_marker_params_still_fill_the_gaps_a_tier_leaves_open(self): + router = Router(model_list=[self._region_marker_entry(), self._bedrock_tier_entry()]) + + sent = await self._routed_call_kwargs(router) + + assert sent["aws_region_name"] == "eu-west-3" + assert sent["drop_params"] is True + + @pytest.mark.asyncio + async def test_request_supplied_param_beats_both_the_marker_and_the_tier(self): + router = Router(model_list=[self._region_marker_entry(), self._bedrock_tier_entry(aws_region_name="us-east-1")]) + + sent = await self._routed_call_kwargs(router, aws_region_name="ap-south-1") + + assert sent["aws_region_name"] == "ap-south-1" + + @pytest.mark.asyncio + async def test_forwarded_params_keep_yielding_to_the_deployment_on_later_routing_passes(self): + """A retry or fallback re-enters the hook with the first pass's forwarded params + already in the kwargs; they must still yield to the next deployment's own values.""" + router = Router( + model_list=[ + self._region_marker_entry(), + self._bedrock_tier_entry(), + self._bedrock_tier_entry(model_name="pinned-tier", aws_region_name="us-east-1"), + ] + ) + request_kwargs: Dict = {} + messages = [{"role": "user", "content": "hi"}] + + await router.async_pre_routing_hook(model="smart-router", request_kwargs=request_kwargs, messages=messages) + router._update_kwargs_with_deployment( + deployment=router.get_model_list(model_name="bedrock-tier")[0], kwargs=request_kwargs + ) + assert request_kwargs["aws_region_name"] == "eu-west-3" + + await router.async_pre_routing_hook(model="smart-router", request_kwargs=request_kwargs, messages=messages) + router._update_kwargs_with_deployment( + deployment=router.get_model_list(model_name="pinned-tier")[0], kwargs=request_kwargs + ) + assert "aws_region_name" not in request_kwargs + assert request_kwargs["drop_params"] is True + class TestAdaptiveSoftFloors: def test_adaptive_defaults_use_cost_weighted_cold_policy(self): From 7683affdfff578ec2b6e7130c17934656e0d4174 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 20 Aug 2026 03:23:28 -0700 Subject: [PATCH 2/2] fix(router): carry forwarded auto_router marker params per request and keep tier overrides and marker flags The forwarded-keys record moves off the shared metadata dict onto the per-request kwargs, where _update_kwargs_with_deployment consumes it, so sibling requests that share a metadata dict (abatch_completion) can no longer clear it mid-routing. Per-tier litellm_params from the hook response are never treated as forwarded marker params, and a deployment only beats a forwarded value when it sets its own, not when it carries a LiteLLM_Params default such as merge_reasoning_content_in_choices=False. --- litellm/constants.py | 1 - litellm/proxy/common_utils/callback_utils.py | 2 - litellm/proxy/litellm_pre_call_utils.py | 2 - litellm/router.py | 61 ++++---- .../router_strategy/test_complexity_router.py | 140 +++++++++++++----- 5 files changed, 135 insertions(+), 71 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index ea1c3f9ecee..facfc6f7c19 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1329,7 +1329,6 @@ OLD_LITELLM_METADATA_FIELD: Final = "metadata" RETURN_RAW_MODEL_NAME_METADATA_KEY: Final = "_complexity_router_return_raw_model_name" SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY: Final = "_session_deployment_affinity_ttl" CONSUMED_REQUEST_TAGS_METADATA_KEY: Final = "_consumed_request_tags" -ALIAS_MARKER_FORWARDED_PARAMS_METADATA_KEY: Final = "_alias_marker_forwarded_params" INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin" LITELLM_TRUNCATED_PAYLOAD_FIELD: Final = "litellm_truncated" LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE: Final = ( diff --git a/litellm/proxy/common_utils/callback_utils.py b/litellm/proxy/common_utils/callback_utils.py index b6a0683c0ff..9379a8577a3 100644 --- a/litellm/proxy/common_utils/callback_utils.py +++ b/litellm/proxy/common_utils/callback_utils.py @@ -10,7 +10,6 @@ import litellm from litellm import get_secret from litellm._logging import verbose_proxy_logger from litellm.constants import ( - ALIAS_MARKER_FORWARDED_PARAMS_METADATA_KEY, CONSUMED_REQUEST_TAGS_METADATA_KEY, PRE_CALL_EXECUTED_GUARDRAILS_KEY, SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, @@ -508,7 +507,6 @@ LITELLM_PROXY_INTERNAL_METADATA_KEYS: Final = frozenset( PRE_CALL_EXECUTED_GUARDRAILS_KEY, SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, CONSUMED_REQUEST_TAGS_METADATA_KEY, - ALIAS_MARKER_FORWARDED_PARAMS_METADATA_KEY, "disable_global_guardrails", "disable_global_guardrail", "opted_out_global_guardrails", diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 590852d5e8d..2ec5c34958c 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -16,7 +16,6 @@ import litellm from litellm._logging import verbose_logger, verbose_proxy_logger from litellm._service_logger import ServiceLogging from litellm.constants import ( - ALIAS_MARKER_FORWARDED_PARAMS_METADATA_KEY, CONSUMED_REQUEST_TAGS_METADATA_KEY, INTERNAL_CALL_ORIGIN_METADATA_KEY, LITELLM_PROXY_MASTER_KEY_ALIAS, @@ -268,7 +267,6 @@ _UNTRUSTED_METADATA_CONTROL_FIELDS: Final = ( "routing_decision", SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, CONSUMED_REQUEST_TAGS_METADATA_KEY, - ALIAS_MARKER_FORWARDED_PARAMS_METADATA_KEY, INTERNAL_CALL_ORIGIN_METADATA_KEY, "standard_logging_object", "proxy_server_request", diff --git a/litellm/router.py b/litellm/router.py index f8fc5bede8e..e4e3a857411 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -44,7 +44,6 @@ from litellm.caching.caching import ( RedisClusterCache, ) from litellm.constants import ( - ALIAS_MARKER_FORWARDED_PARAMS_METADATA_KEY, CONSUMED_REQUEST_TAGS_METADATA_KEY, DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS, DEFAULT_HEALTH_CHECK_INTERVAL, @@ -324,6 +323,7 @@ def model_info_is_active_for_environment(model_info: Mapping[str, object] | None _PreRoutingStrategyT = TypeVar("_PreRoutingStrategyT") _ALIAS_PARAMS_NEVER_FORWARDED: Final = frozenset({"model", "api_base", "api_key", "api_version"}) +_ALIAS_MARKER_FORWARDED_PARAMS_KWARG: Final = "_alias_marker_forwarded_params" def _stream_chunks_have_generated_content(chunks: Sequence[ModelResponseStream]) -> bool: @@ -3240,7 +3240,9 @@ class Router: - Adds default litellm params to kwargs, if set. - Merges tools from deployment with request (proxy-configured tools + request tools). """ - for key in self._forwarded_alias_marker_keys_the_deployment_sets(deployment=deployment, kwargs=kwargs): + for key in self._forwarded_alias_marker_keys_the_deployment_sets( + deployment=deployment, forwarded_keys=kwargs.pop(_ALIAS_MARKER_FORWARDED_PARAMS_KWARG, ()) + ): kwargs.pop(key, None) self._merge_tools_from_deployment(deployment=deployment, kwargs=kwargs) @@ -11567,11 +11569,6 @@ class Router: self._stamp_or_clear_metadata_key( request_kwargs=request_kwargs, key=CONSUMED_REQUEST_TAGS_METADATA_KEY, value=None ) - self._stamp_or_clear_metadata_key( - request_kwargs=request_kwargs, - key=ALIAS_MARKER_FORWARDED_PARAMS_METADATA_KEY, - value=self._still_forwarded_alias_marker_keys(request_kwargs) or None, - ) return None pre_routing_hook_response: Final = await selected_strategy.strategy.async_pre_routing_hook( @@ -11613,22 +11610,27 @@ class Router: # excluded here: they price the alias, not the deployment the hook # selected, and forwarding them re-registers the routed deployment at # the alias's price (an explicit 0 makes every alias request bill $0). - # Forwarded params only fill gaps: the keys inserted here are stamped on the - # request so `_update_kwargs_with_deployment` can drop any the selected + # Forwarded params only fill gaps: the keys inserted here ride along on the + # request (top level, so sibling requests sharing a `metadata` dict never see + # them) until `_update_kwargs_with_deployment` drops any the selected # deployment sets itself (its own `aws_region_name` beats the marker's). + # Per-tier `litellm_params` on the hook response are deliberate overrides + # the caller applies on top, so those keys are never forwarded here. marker_params: Final = ( self._forwardable_alias_marker_params(model=model, strategy_tags=selected_strategy.tags) if pre_routing_hook_response is not None else () ) - still_forwarded: Final = self._still_forwarded_alias_marker_keys(request_kwargs) - newly_forwarded: Final = tuple((key, value) for key, value in marker_params if key not in request_kwargs) - request_kwargs.update(newly_forwarded) - self._stamp_or_clear_metadata_key( - request_kwargs=request_kwargs, - key=ALIAS_MARKER_FORWARDED_PARAMS_METADATA_KEY, - value=(*still_forwarded, *(key for key, _ in newly_forwarded)) or None, + tier_param_keys: Final = ( + tuple(pre_routing_hook_response.litellm_params or ()) if pre_routing_hook_response is not None else () ) + newly_forwarded: Final = tuple( + (key, value) for key, value in marker_params if key not in request_kwargs and key not in tier_param_keys + ) + request_kwargs.pop(_ALIAS_MARKER_FORWARDED_PARAMS_KWARG, None) + request_kwargs.update(newly_forwarded) + if newly_forwarded: + request_kwargs.update(((_ALIAS_MARKER_FORWARDED_PARAMS_KWARG, tuple(key for key, _ in newly_forwarded)),)) return pre_routing_hook_response @@ -11655,30 +11657,27 @@ class Router: and value is not None ) - @staticmethod - def _still_forwarded_alias_marker_keys(request_kwargs: Mapping[str, object]) -> tuple[str, ...]: - stamps: Final = tuple( - stamp - for bucket in (request_kwargs.get("litellm_metadata"), request_kwargs.get("metadata")) - if isinstance(bucket, Mapping) - for stamp in (bucket.get(ALIAS_MARKER_FORWARDED_PARAMS_METADATA_KEY),) - if isinstance(stamp, (list, tuple)) - ) - return tuple(key for stamp in stamps for key in stamp if isinstance(key, str) and key in request_kwargs) - @staticmethod def _forwarded_alias_marker_keys_the_deployment_sets( - deployment: Mapping[str, object], kwargs: Mapping[str, object] + deployment: Mapping[str, object], forwarded_keys: object ) -> tuple[str, ...]: deployment_litellm_params: Final = deployment.get("litellm_params") - if not isinstance(deployment_litellm_params, Mapping): + if not isinstance(deployment_litellm_params, Mapping) or not isinstance(forwarded_keys, tuple): return () return tuple( key - for key in Router._still_forwarded_alias_marker_keys(kwargs) - if deployment_litellm_params.get(key) is not None + for key in forwarded_keys + if isinstance(key, str) and Router._deployment_sets_litellm_param(deployment_litellm_params, key) ) + @staticmethod + def _deployment_sets_litellm_param(deployment_litellm_params: Mapping[str, object], key: str) -> bool: + value: Final = deployment_litellm_params.get(key) + if value is None: + return False + field: Final = LiteLLM_Params.model_fields.get(key) + return field is None or value != field.default + def _consumed_request_tags_stamp( self, selected_strategy: "TaggedPreRoutingStrategy[PreRoutingStrategy]", diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index ac3b1089e3f..4b9d3d7bfff 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -278,8 +278,7 @@ class TestReasoningMarkerScoring: def test_reasoning_override_applies_at_the_simple_medium_boundary(self, complexity_router): """A score sitting exactly on simple_medium is not SIMPLE, so the override still promotes it.""" prompt = ( - "Give me the pros and cons, step by step, of moving our checkout service " - "to an event-driven architecture." + "Give me the pros and cons, step by step, of moving our checkout service to an event-driven architecture." ) tier, score, signals = complexity_router.classify(prompt) assert score == complexity_router.config.tier_boundaries["simple_medium"] @@ -299,8 +298,7 @@ class TestReasoningMarkerScoring: def test_floor_defaults_to_simple_medium_and_follows_it(self, mock_router_instance, basic_config): """Unset tracks simple_medium, so moving that boundary moves the floor with it.""" prompt = ( - "Give me the pros and cons, step by step, of moving our checkout service " - "to an event-driven architecture." + "Give me the pros and cons, step by step, of moving our checkout service to an event-driven architecture." ) low = ComplexityRouter( model_name="test-complexity-router", @@ -320,8 +318,7 @@ class TestReasoningMarkerScoring: def test_explicit_floor_overrides_the_boundary(self, mock_router_instance, basic_config): """A configured floor decides the override, not simple_medium.""" prompt = ( - "Give me the pros and cons, step by step, of moving our checkout service " - "to an event-driven architecture." + "Give me the pros and cons, step by step, of moving our checkout service to an event-driven architecture." ) router = ComplexityRouter( model_name="test-complexity-router", @@ -340,8 +337,7 @@ class TestReasoningMarkerScoring: def test_configured_floor_is_applied_with_greater_or_equal(self, mock_router_instance, basic_config): """A score landing exactly on the configured floor still promotes.""" prompt = ( - "Give me the pros and cons, step by step, of moving our checkout service " - "to an event-driven architecture." + "Give me the pros and cons, step by step, of moving our checkout service to an event-driven architecture." ) router = ComplexityRouter( model_name="test-complexity-router", @@ -2523,11 +2519,15 @@ class TestRouterPreRoutingSharedAliasName: } @staticmethod - def _bedrock_tier_entry(model_name: str = "bedrock-tier", aws_region_name: str | None = None) -> dict: + def _bedrock_tier_entry( + model_name: str = "bedrock-tier", + aws_region_name: str | None = None, + model: str = "bedrock/us.anthropic.claude-sonnet-5", + ) -> dict: return { "model_name": model_name, "litellm_params": { - "model": "bedrock/us.anthropic.claude-sonnet-5", + "model": model, **({"aws_region_name": aws_region_name} if aws_region_name else {}), }, } @@ -2571,31 +2571,102 @@ class TestRouterPreRoutingSharedAliasName: assert sent["aws_region_name"] == "ap-south-1" @pytest.mark.asyncio - async def test_forwarded_params_keep_yielding_to_the_deployment_on_later_routing_passes(self): - """A retry or fallback re-enters the hook with the first pass's forwarded params - already in the kwargs; they must still yield to the next deployment's own values.""" + async def test_complexity_tier_litellm_params_beat_the_tier_deployments_own_params(self): + """Per-tier `litellm_params` are deliberate overrides, not forwarded marker params: + they keep winning over the tier deployment's own value.""" + marker = self._region_marker_entry() + marker["litellm_params"]["complexity_router_config"] = { + "tiers": { + tier: {"model_name": "bedrock-tier", "litellm_params": {"aws_region_name": "us-west-2"}} + for tier in ("SIMPLE", "MEDIUM", "COMPLEX", "REASONING") + } + } + router = Router(model_list=[marker, self._bedrock_tier_entry(aws_region_name="us-east-1")]) + + sent = await self._routed_call_kwargs(router) + + assert sent["aws_region_name"] == "us-west-2" + assert sent["drop_params"] is True + + @pytest.mark.asyncio + async def test_a_markers_explicit_flag_beats_the_tiers_pydantic_default(self): + """Every deployment materializes `LiteLLM_Params` defaults such as + `merge_reasoning_content_in_choices: False`; a default is not the tier setting its own value.""" + marker = self._region_marker_entry() + marker["litellm_params"]["merge_reasoning_content_in_choices"] = True + router = Router(model_list=[marker, self._bedrock_tier_entry()]) + + sent = await self._routed_call_kwargs(router) + + assert sent["merge_reasoning_content_in_choices"] is True + + @pytest.mark.asyncio + async def test_sibling_request_sharing_the_metadata_dict_cannot_unpin_the_tier(self): + """`abatch_completion` hands every per-model task the same `metadata` dict; a plain + group's routing pass interleaving with the auto-router's must not leak the marker's region.""" router = Router( model_list=[ self._region_marker_entry(), - self._bedrock_tier_entry(), - self._bedrock_tier_entry(model_name="pinned-tier", aws_region_name="us-east-1"), + self._bedrock_tier_entry(aws_region_name="us-east-1"), + self._bedrock_tier_entry( + model_name="plain", aws_region_name="us-west-2", model="bedrock/us.anthropic.claude-haiku-5" + ), ] ) - request_kwargs: Dict = {} - messages = [{"role": "user", "content": "hi"}] + healthy_deployments = router.async_get_healthy_deployments - await router.async_pre_routing_hook(model="smart-router", request_kwargs=request_kwargs, messages=messages) - router._update_kwargs_with_deployment( - deployment=router.get_model_list(model_name="bedrock-tier")[0], kwargs=request_kwargs - ) - assert request_kwargs["aws_region_name"] == "eu-west-3" + async def yield_between_routing_and_dispatch(*args, **kwargs): + await asyncio.sleep(0.01) + return await healthy_deployments(*args, **kwargs) - await router.async_pre_routing_hook(model="smart-router", request_kwargs=request_kwargs, messages=messages) - router._update_kwargs_with_deployment( - deployment=router.get_model_list(model_name="pinned-tier")[0], kwargs=request_kwargs - ) - assert "aws_region_name" not in request_kwargs - assert request_kwargs["drop_params"] is True + sent: Dict[str, str | None] = {} + + async def record(**kwargs): + sent[kwargs["model"]] = kwargs.get("aws_region_name") + return litellm.ModelResponse(choices=[{"message": {"content": "hi"}}]) + + with ( + patch.object(router, "async_get_healthy_deployments", yield_between_routing_and_dispatch), + patch.object(litellm, "acompletion", AsyncMock(side_effect=record)), + ): + await router.abatch_completion( + models=["smart-router", "plain"], + messages=[{"role": "user", "content": "hi"}], + metadata={"shared": True}, + ) + + assert sent == { + "bedrock/us.anthropic.claude-sonnet-5": "us-east-1", + "bedrock/us.anthropic.claude-haiku-5": "us-west-2", + } + + @pytest.mark.asyncio + async def test_routing_leaves_no_forwarded_keys_record_on_the_provider_call(self): + router = Router(model_list=[self._region_marker_entry(), self._bedrock_tier_entry()]) + + sent = await self._routed_call_kwargs(router) + + assert not any(key.startswith("_alias_marker") for key in sent) + + def test_forwarded_alias_marker_keys_the_deployment_sets(self): + deployment = {"litellm_params": {"model": "bedrock/x", "aws_region_name": "us-east-1", "timeout": None}} + + assert Router._forwarded_alias_marker_keys_the_deployment_sets( + deployment=deployment, forwarded_keys=("aws_region_name", "timeout", "drop_params") + ) == ("aws_region_name",) + assert Router._forwarded_alias_marker_keys_the_deployment_sets(deployment=deployment, forwarded_keys=()) == () + assert Router._forwarded_alias_marker_keys_the_deployment_sets(deployment=deployment, forwarded_keys=None) == () + assert Router._forwarded_alias_marker_keys_the_deployment_sets(deployment={}, forwarded_keys=("x",)) == () + + def test_deployment_sets_litellm_param(self): + params = {"aws_region_name": "us-east-1", "timeout": None, "use_litellm_proxy": False, "custom_flag": False} + + assert Router._deployment_sets_litellm_param(params, "aws_region_name") is True + assert Router._deployment_sets_litellm_param(params, "timeout") is False + assert Router._deployment_sets_litellm_param(params, "missing") is False + assert Router._deployment_sets_litellm_param(params, "use_litellm_proxy") is False + assert Router._deployment_sets_litellm_param({"use_litellm_proxy": True}, "use_litellm_proxy") is True + assert Router._deployment_sets_litellm_param(params, "custom_flag") is True class TestAdaptiveSoftFloors: @@ -7395,6 +7466,8 @@ class TestClassificationRubrics: }, ) assert config.classifier_llm_config.system_prompt == "Grade the data sensitivity of the request." + + def _custom_tier_config(**overrides) -> Dict: """A valid operator-defined tier set: two built-in names plus one custom tier.""" return { @@ -8243,6 +8316,8 @@ class TestPlanModeTierFloor: assert result.model == "gpt-4o" assert result.routing_decision is not None assert result.routing_decision["tier"] == "MEDIUM" + + def test_tier_model_params_are_normalized_without_changing_model_pools(): config = ComplexityRouterConfig( tiers={ @@ -8353,13 +8428,8 @@ async def test_tier_model_params_reach_the_hook_response_and_override_client_val async def test_tier_params_mask_credentials_in_routing_decision(route, mock_router_instance): params = {"reasoning_effort": "xhigh", "api_key": "secret-tier-key"} config = { - "tiers": { - tier.value: {"model_name": "opus", "litellm_params": params} - for tier in ComplexityTier - }, - "keyword_tier_rules": [{"keywords": ["reason carefully"], "tier": "REASONING"}] - if route == "keyword" - else None, + "tiers": {tier.value: {"model_name": "opus", "litellm_params": params} for tier in ComplexityTier}, + "keyword_tier_rules": [{"keywords": ["reason carefully"], "tier": "REASONING"}] if route == "keyword" else None, "session_affinity": route == "session", } router = ComplexityRouter(